From ab3fe0da81f8bc8777019dc879074b267bd5bb68 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Mon, 12 Jun 2023 17:37:03 +0530 Subject: [PATCH 001/222] refractor(classification): datahub classifier init (#8193) --- .../ingestion/glossary/datahub_classifier.py | 4 -- .../integration/snowflake/test_snowflake.py | 48 ++++++++++--------- .../test_snowflake_legacy_lineage.py | 48 ++++++++++--------- 3 files changed, 50 insertions(+), 50 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/glossary/datahub_classifier.py b/metadata-ingestion/src/datahub/ingestion/glossary/datahub_classifier.py index 3ab3faa769ae2..575289c0473a6 100644 --- a/metadata-ingestion/src/datahub/ingestion/glossary/datahub_classifier.py +++ b/metadata-ingestion/src/datahub/ingestion/glossary/datahub_classifier.py @@ -86,22 +86,18 @@ class Config: class DataHubClassifierConfig(ConfigModel): confidence_level_threshold: float = Field( default=0.68, - init=False, description="The confidence threshold above which the prediction is considered as a proposal", ) info_types: Optional[List[str]] = Field( default=None, - init=False, description="List of infotypes to be predicted. By default, all supported infotypes are considered, along with any custom infotypes configured in `info_types_config`.", ) info_types_config: Dict[str, InfoTypeConfig] = Field( default=DEFAULT_CLASSIFIER_CONFIG, - init=False, description="Configuration details for infotypes. See [reference_input.py](https://github.com/acryldata/datahub-classify/blob/main/datahub-classify/src/datahub_classify/reference_input.py) for default configuration.", ) minimum_values_threshold: int = Field( default=50, - init=False, description="Minimum number of non-null column values required to process `values` prediction factor.", ) diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py index 2b8242800ba9b..62c47821209d2 100644 --- a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py +++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py @@ -76,36 +76,38 @@ def test_snowflake_basic(pytestconfig, tmp_path, mock_time, mock_datahub_graph): mock_sample_values.return_value = pd.DataFrame( data={ - "col_1": [random.randint(0, 100) for i in range(20)], + "col_1": [random.randint(1, 80) for i in range(20)], "col_2": [random_email() for i in range(20)], "col_3": [random_cloud_region() for i in range(20)], } ) - datahub_classifier_config = DataHubClassifierConfig() - datahub_classifier_config.confidence_level_threshold = 0.58 - datahub_classifier_config.minimum_values_threshold = 10 - datahub_classifier_config.info_types_config = { - "Age": InfoTypeConfig( - Prediction_Factors_and_Weights=PredictionFactorsAndWeights( - Name=0, Values=1, Description=0, Datatype=0 - ) - ), - "CloudRegion": InfoTypeConfig( - Prediction_Factors_and_Weights=PredictionFactorsAndWeights( - Name=0, - Description=0, - Datatype=0, - Values=1, + datahub_classifier_config = DataHubClassifierConfig( + minimum_values_threshold=10, + confidence_level_threshold=0.58, + info_types_config={ + "Age": InfoTypeConfig( + Prediction_Factors_and_Weights=PredictionFactorsAndWeights( + Name=0, Values=1, Description=0, Datatype=0 + ) ), - Values=ValuesFactorConfig( - prediction_type="regex", - regex=[ - r"(af|ap|ca|eu|me|sa|us)-(central|north|(north(?:east|west))|south|south(?:east|west)|east|west)-\d+" - ], + "CloudRegion": InfoTypeConfig( + Prediction_Factors_and_Weights=PredictionFactorsAndWeights( + Name=0, + Description=0, + Datatype=0, + Values=1, + ), + Values=ValuesFactorConfig( + prediction_type="regex", + regex=[ + r"(af|ap|ca|eu|me|sa|us)-(central|north|(north(?:east|west))|south|south(?:east|west)|east|west)-\d+" + ], + ), ), - ), - } + }, + ) + pipeline = Pipeline( config=PipelineConfig( source=SourceConfig( diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake_legacy_lineage.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake_legacy_lineage.py index afd50930c5a93..55dde8514d1f7 100644 --- a/metadata-ingestion/tests/integration/snowflake/test_snowflake_legacy_lineage.py +++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake_legacy_lineage.py @@ -50,36 +50,38 @@ def test_snowflake_basic(pytestconfig, tmp_path, mock_time, mock_datahub_graph): mock_sample_values.return_value = pd.DataFrame( data={ - "col_1": [random.randint(0, 100) for i in range(20)], + "col_1": [random.randint(1, 80) for i in range(20)], "col_2": [random_email() for i in range(20)], "col_3": [random_cloud_region() for i in range(20)], } ) - datahub_classifier_config = DataHubClassifierConfig() - datahub_classifier_config.confidence_level_threshold = 0.58 - datahub_classifier_config.minimum_values_threshold = 10 - datahub_classifier_config.info_types_config = { - "Age": InfoTypeConfig( - Prediction_Factors_and_Weights=PredictionFactorsAndWeights( - Name=0, Values=1, Description=0, Datatype=0 - ) - ), - "CloudRegion": InfoTypeConfig( - Prediction_Factors_and_Weights=PredictionFactorsAndWeights( - Name=0, - Description=0, - Datatype=0, - Values=1, + datahub_classifier_config = DataHubClassifierConfig( + minimum_values_threshold=10, + confidence_level_threshold=0.58, + info_types_config={ + "Age": InfoTypeConfig( + Prediction_Factors_and_Weights=PredictionFactorsAndWeights( + Name=0, Values=1, Description=0, Datatype=0 + ) ), - Values=ValuesFactorConfig( - prediction_type="regex", - regex=[ - r"(af|ap|ca|eu|me|sa|us)-(central|north|(north(?:east|west))|south|south(?:east|west)|east|west)-\d+" - ], + "CloudRegion": InfoTypeConfig( + Prediction_Factors_and_Weights=PredictionFactorsAndWeights( + Name=0, + Description=0, + Datatype=0, + Values=1, + ), + Values=ValuesFactorConfig( + prediction_type="regex", + regex=[ + r"(af|ap|ca|eu|me|sa|us)-(central|north|(north(?:east|west))|south|south(?:east|west)|east|west)-\d+" + ], + ), ), - ), - } + }, + ) + pipeline = Pipeline( config=PipelineConfig( source=SourceConfig( From f3cf9b7d5a0022b510a4c9ae33e9664ec0f45a63 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Mon, 12 Jun 2023 17:38:20 +0530 Subject: [PATCH 002/222] fix(glue): fix typo in reported warning, report with flow_urn (#8138) --- .../src/datahub/ingestion/source/aws/glue.py | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py index f3380fa814b0d..eeba0d8649ca2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py @@ -304,7 +304,9 @@ def get_all_jobs(self): return jobs - def get_dataflow_graph(self, script_path: str) -> Optional[Dict[str, Any]]: + def get_dataflow_graph( + self, script_path: str, flow_urn: str + ) -> Optional[Dict[str, Any]]: """ Get the DAG of transforms and data sources/sinks for a job. @@ -320,8 +322,8 @@ def get_dataflow_graph(self, script_path: str) -> Optional[Dict[str, Any]]: # catch any other cases where the script path is invalid if not script_path.startswith("s3://"): - self.report.report_warning( - script_path, + self.report_warning( + flow_urn, f"Error parsing DAG for Glue job. The script {script_path} is not a valid S3 path.", ) self.report.num_job_script_location_invalid += 1 @@ -338,8 +340,8 @@ def get_dataflow_graph(self, script_path: str) -> Optional[Dict[str, Any]]: try: obj = self.s3_client.get_object(Bucket=bucket, Key=key) except botocore.exceptions.ClientError as e: - self.report.report_failure( - script_path, + self.report_warning( + flow_urn, f"Unable to download DAG for Glue job from {script_path}, so job subtasks and lineage will be missing: {e}", ) self.report.num_job_script_failed_download += 1 @@ -353,8 +355,8 @@ def get_dataflow_graph(self, script_path: str) -> Optional[Dict[str, Any]]: # sometimes the Python script can be user-modified and the script is not valid for graph extraction except self.glue_client.exceptions.InvalidInputException as e: - self.report.report_warning( - script_path, + self.report_warning( + flow_urn, f"Error parsing DAG for Glue job. The script {script_path} cannot be processed by Glue (this usually occurs when it has been user-modified): {e}", ) self.report.num_job_script_failed_parsing += 1 @@ -425,9 +427,9 @@ def process_dataflow_node( s3_uri = self.get_s3_uri(node_args) if s3_uri is None: - self.report.report_warning( - f"{node['Nodetype']}-{node['Id']}", - f"Could not find script path for job {node['Nodetype']}-{node['Id']} in flow {flow_urn}. Skipping", + self.report_warning( + flow_urn, + f"Could not find script path for job {node['NodeType']}-{node['Id']} in flow {flow_urn}. Skipping", ) return None @@ -461,12 +463,9 @@ def process_dataflow_node( else: if self.source_config.ignore_unsupported_connectors: - logger.debug( - f"Unrecognized node {node['Nodetype']}-{node['Id']} in flow {flow_urn}. Args : {node_args}", - ) - self.report.report_warning( - f"{node['Nodetype']}-{node['Id']}", - f"Unrecognized node {node['Nodetype']}-{node['Id']} in flow {flow_urn}. Skipping", + self.report_warning( + flow_urn, + f"Unrecognized node {node['NodeType']}-{node['Id']} in flow {flow_urn}. Args: {node_args} Skipping", ) return None else: @@ -528,7 +527,7 @@ def process_dataflow_graph( # Source and Target for some edges is not available # in nodes. this may lead to broken edge in lineage. if source_node is None or target_node is None: - self.report.report_warning( + self.report_warning( flow_urn, f"Unrecognized source or target node in edge: {edge}. Skipping." "This may lead to missing lineage", @@ -992,7 +991,7 @@ def _transform_extraction(self) -> Iterable[MetadataWorkUnit]: dag: Optional[Dict[str, Any]] = None if job_script_location is not None: - dag = self.get_dataflow_graph(job_script_location) + dag = self.get_dataflow_graph(job_script_location, flow_urn) else: self.report.num_job_script_location_missing += 1 @@ -1213,3 +1212,7 @@ def get_data_platform_instance() -> DataPlatformInstanceClass: def get_report(self): return self.report + + def report_warning(self, key: str, reason: str) -> None: + logger.warning(f"{key}: {reason}") + self.report.report_warning(key, reason) From 8e8f2f5e654e62efb458ff639ac2f8189b004eaa Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Mon, 12 Jun 2023 20:32:04 +0530 Subject: [PATCH 003/222] fix(ingest/delta-lake): fix CI issues due to delta lake version bump (#8215) --- .../source/delta_lake/delta_lake_utils.py | 25 ++++++++++++++++--- .../ingestion/source/delta_lake/source.py | 4 +-- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/delta_lake_utils.py b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/delta_lake_utils.py index 9738dcc311021..15b1c9006f9a9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/delta_lake_utils.py +++ b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/delta_lake_utils.py @@ -1,7 +1,20 @@ import pathlib from typing import Dict, Optional -from deltalake import DeltaTable, PyDeltaTableError +from deltalake import DeltaTable + +try: + from deltalake.exceptions import TableNotFoundError + + DELTALAKE_VERSION_GTE_0_10_0 = True + + +except ImportError: + # For deltalake < 0.10.0 + from deltalake import PyDeltaTableError # type: ignore[attr-defined] + + DELTALAKE_VERSION_GTE_0_10_0 = False + from datahub.ingestion.source.delta_lake.config import DeltaLakeSourceConfig @@ -20,8 +33,14 @@ def read_delta_table( storage_options=opts, without_files=not delta_lake_config.require_files, ) - except PyDeltaTableError as e: - if "Not a Delta table" not in str(e): + except Exception as e: + if (DELTALAKE_VERSION_GTE_0_10_0 and isinstance(e, TableNotFoundError)) or ( + not DELTALAKE_VERSION_GTE_0_10_0 + and isinstance(e, PyDeltaTableError) + and "Not a Delta table" in str(e) + ): + pass + else: raise e return None diff --git a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py index 1a1fd589ded2e..0c0cbaf3b4803 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py @@ -159,7 +159,7 @@ def _create_operation_aspect_wu( reported_time: int = int(time.time() * 1000) last_updated_timestamp: int = hist["timestamp"] statement_type = OPERATION_STATEMENT_TYPES.get( - hist.get("operation"), OperationTypeClass.CUSTOM + hist.get("operation", "UNKNOWN"), OperationTypeClass.CUSTOM ) custom_type = ( hist.get("operation") @@ -276,7 +276,7 @@ def ingest_table( if s3_tags is not None: dataset_snapshot.aspects.append(s3_tags) mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot) - yield MetadataWorkUnit(id=delta_table.metadata().id, mce=mce) + yield MetadataWorkUnit(id=str(delta_table.metadata().id), mce=mce) yield from self.container_WU_creator.create_container_hierarchy( browse_path, dataset_urn From 166dc90896f163418d9c601bb4ecc28285fe95b4 Mon Sep 17 00:00:00 2001 From: Jinlin Yang <86577891+jinlintt@users.noreply.github.com> Date: Tue, 13 Jun 2023 05:57:54 -0700 Subject: [PATCH 004/222] Upgrade kafka and its dependencies to 3.4 in docker compose (#8161) Co-authored-by: Pedro Silva --- docker/docker-compose-with-cassandra.yml | 6 +++--- docker/docker-compose-without-neo4j.yml | 6 +++--- docker/docker-compose.tools.yml | 2 +- docker/docker-compose.yml | 6 +++--- docker/quickstart/docker-compose-m1.quickstart.yml | 6 +++--- .../docker-compose-without-neo4j-m1.quickstart.yml | 6 +++--- .../quickstart/docker-compose-without-neo4j.quickstart.yml | 6 +++--- docker/quickstart/docker-compose.quickstart.yml | 6 +++--- .../test/java/datahub/client/kafka/containers/Utils.java | 2 +- 9 files changed, 23 insertions(+), 23 deletions(-) diff --git a/docker/docker-compose-with-cassandra.yml b/docker/docker-compose-with-cassandra.yml index 309be76a5a5a1..87cd104cd9cd5 100644 --- a/docker/docker-compose-with-cassandra.yml +++ b/docker/docker-compose-with-cassandra.yml @@ -148,7 +148,7 @@ services: schema-registry: container_name: schema-registry hostname: schema-registry - image: confluentinc/cp-schema-registry:7.2.5 + image: confluentinc/cp-schema-registry:7.4.0 ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 env_file: schema-registry/env/docker.env @@ -164,7 +164,7 @@ services: broker: container_name: broker hostname: broker - image: confluentinc/cp-kafka:7.2.5 + image: confluentinc/cp-kafka:7.4.0 ports: - 29092:29092 - 9092:9092 @@ -183,7 +183,7 @@ services: zookeeper: container_name: zookeeper hostname: zookeeper - image: confluentinc/cp-zookeeper:7.2.5 + image: confluentinc/cp-zookeeper:7.4.0 ports: - 2181:2181 env_file: zookeeper/env/docker.env diff --git a/docker/docker-compose-without-neo4j.yml b/docker/docker-compose-without-neo4j.yml index 8abee9632f56d..87ec6105d8298 100644 --- a/docker/docker-compose-without-neo4j.yml +++ b/docker/docker-compose-without-neo4j.yml @@ -125,7 +125,7 @@ services: schema-registry: container_name: schema-registry hostname: schema-registry - image: confluentinc/cp-schema-registry:7.2.5 + image: confluentinc/cp-schema-registry:7.4.0 ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 env_file: schema-registry/env/docker.env @@ -141,7 +141,7 @@ services: broker: container_name: broker hostname: broker - image: confluentinc/cp-kafka:7.2.5 + image: confluentinc/cp-kafka:7.4.0 ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 env_file: broker/env/docker.env @@ -159,7 +159,7 @@ services: zookeeper: container_name: zookeeper hostname: zookeeper - image: confluentinc/cp-zookeeper:7.2.5 + image: confluentinc/cp-zookeeper:7.4.0 ports: - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 env_file: zookeeper/env/docker.env diff --git a/docker/docker-compose.tools.yml b/docker/docker-compose.tools.yml index 346295a1f2a31..99e36d8ba4511 100644 --- a/docker/docker-compose.tools.yml +++ b/docker/docker-compose.tools.yml @@ -3,7 +3,7 @@ version: '3.8' services: kafka-rest-proxy: - image: confluentinc/cp-kafka-rest:7.2.5 + image: confluentinc/cp-kafka-rest:7.4.0 env_file: kafka-rest-proxy/env/docker.env hostname: kafka-rest-proxy container_name: kafka-rest-proxy diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 5f5ea705ed327..b2c55fafdb26b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -144,7 +144,7 @@ services: schema-registry: container_name: schema-registry hostname: schema-registry - image: confluentinc/cp-schema-registry:7.2.5 + image: confluentinc/cp-schema-registry:7.4.0 ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 env_file: schema-registry/env/docker.env @@ -160,7 +160,7 @@ services: broker: container_name: broker hostname: broker - image: confluentinc/cp-kafka:7.2.5 + image: confluentinc/cp-kafka:7.4.0 ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 env_file: broker/env/docker.env @@ -178,7 +178,7 @@ services: zookeeper: container_name: zookeeper hostname: zookeeper - image: confluentinc/cp-zookeeper:7.2.5 + image: confluentinc/cp-zookeeper:7.4.0 ports: - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 env_file: zookeeper/env/docker.env diff --git a/docker/quickstart/docker-compose-m1.quickstart.yml b/docker/quickstart/docker-compose-m1.quickstart.yml index 73e176c2c6eff..ab54123fbe646 100644 --- a/docker/quickstart/docker-compose-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-m1.quickstart.yml @@ -23,7 +23,7 @@ services: test: nc -z broker $${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092} timeout: 5s hostname: broker - image: confluentinc/cp-kafka:7.2.5 + image: confluentinc/cp-kafka:7.4.0 ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 volumes: @@ -273,7 +273,7 @@ services: test: nc -z schema-registry ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081} timeout: 5s hostname: schema-registry - image: confluentinc/cp-schema-registry:7.2.5 + image: confluentinc/cp-schema-registry:7.4.0 ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 zookeeper: @@ -288,7 +288,7 @@ services: test: echo srvr | nc zookeeper $${DATAHUB_MAPPED_ZK_PORT:-2181} timeout: 5s hostname: zookeeper - image: confluentinc/cp-zookeeper:7.2.5 + image: confluentinc/cp-zookeeper:7.4.0 ports: - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index 1a260d78bb3f7..7233702fdef24 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -23,7 +23,7 @@ services: test: nc -z broker $${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092} timeout: 5s hostname: broker - image: confluentinc/cp-kafka:7.2.5 + image: confluentinc/cp-kafka:7.4.0 ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 volumes: @@ -247,7 +247,7 @@ services: test: nc -z schema-registry ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081} timeout: 5s hostname: schema-registry - image: confluentinc/cp-schema-registry:7.2.5 + image: confluentinc/cp-schema-registry:7.4.0 ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 zookeeper: @@ -262,7 +262,7 @@ services: test: echo srvr | nc zookeeper $${DATAHUB_MAPPED_ZK_PORT:-2181} timeout: 5s hostname: zookeeper - image: confluentinc/cp-zookeeper:7.2.5 + image: confluentinc/cp-zookeeper:7.4.0 ports: - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index 19d75eed43728..49dab99e383a8 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -23,7 +23,7 @@ services: test: nc -z broker $${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092} timeout: 5s hostname: broker - image: confluentinc/cp-kafka:7.2.5 + image: confluentinc/cp-kafka:7.4.0 ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 volumes: @@ -247,7 +247,7 @@ services: test: nc -z schema-registry ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081} timeout: 5s hostname: schema-registry - image: confluentinc/cp-schema-registry:7.2.5 + image: confluentinc/cp-schema-registry:7.4.0 ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 zookeeper: @@ -262,7 +262,7 @@ services: test: echo srvr | nc zookeeper $${DATAHUB_MAPPED_ZK_PORT:-2181} timeout: 5s hostname: zookeeper - image: confluentinc/cp-zookeeper:7.2.5 + image: confluentinc/cp-zookeeper:7.4.0 ports: - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index b1c9c2d84fbc4..379b6bd1edfa3 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -23,7 +23,7 @@ services: test: nc -z broker $${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092} timeout: 5s hostname: broker - image: confluentinc/cp-kafka:7.2.5 + image: confluentinc/cp-kafka:7.4.0 ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 volumes: @@ -273,7 +273,7 @@ services: test: nc -z schema-registry ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081} timeout: 5s hostname: schema-registry - image: confluentinc/cp-schema-registry:7.2.5 + image: confluentinc/cp-schema-registry:7.4.0 ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 zookeeper: @@ -288,7 +288,7 @@ services: test: echo srvr | nc zookeeper $${DATAHUB_MAPPED_ZK_PORT:-2181} timeout: 5s hostname: zookeeper - image: confluentinc/cp-zookeeper:7.2.5 + image: confluentinc/cp-zookeeper:7.4.0 ports: - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: diff --git a/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/containers/Utils.java b/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/containers/Utils.java index 5078254251433..efa79724f114b 100644 --- a/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/containers/Utils.java +++ b/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/containers/Utils.java @@ -4,7 +4,7 @@ import java.net.ServerSocket; final class Utils { - public static final String CONFLUENT_PLATFORM_VERSION = "7.2.5"; + public static final String CONFLUENT_PLATFORM_VERSION = "7.4.0"; private Utils() { } From 4cfb795c1c3b3669710a136b887fefd841b94e95 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Tue, 13 Jun 2023 15:51:03 +0100 Subject: [PATCH 005/222] chore(release): update default cli for managed ingestion (#8226) --- .../src/app/ingest/source/builder/NameSourceStep.tsx | 2 +- gradle/versioning/versioning.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx b/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx index 48b8a712f06ba..1387934d05f06 100644 --- a/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx @@ -102,7 +102,7 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps) setVersion(event.target.value)} /> diff --git a/gradle/versioning/versioning.gradle b/gradle/versioning/versioning.gradle index 66a51777df1a7..2bbae42fcc5f6 100644 --- a/gradle/versioning/versioning.gradle +++ b/gradle/versioning/versioning.gradle @@ -21,7 +21,7 @@ Produces the following variables and supports token replacement import org.apache.tools.ant.filters.ReplaceTokens def detailedVersionString = "0.0.0-unknown-SNAPSHOT" -def cliMajorVersion = "0.9.5" // base default cli major version +def cliMajorVersion = "0.10.4" // base default cli major version def snapshotVersion = false if (project.hasProperty("releaseVersion")) { version = releaseVersion From 460f96d6838a2c2310ca8a762a5b65812eaedabf Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Tue, 13 Jun 2023 16:23:36 +0100 Subject: [PATCH 006/222] fix(ownership): Corrects graphQL resolver for entity operations (#8219) --- .../domain/CreateDomainResolver.java | 2 +- .../glossary/CreateGlossaryNodeResolver.java | 2 +- .../glossary/CreateGlossaryTermResolver.java | 2 +- .../resolvers/mutate/AddOwnerResolver.java | 2 +- .../resolvers/mutate/util/OwnerUtils.java | 11 +++---- .../resolvers/tag/CreateTagResolver.java | 2 +- .../types/common/mappers/OwnerMapper.java | 2 +- .../common/mappers/OwnerUpdateMapper.java | 10 ++++++- .../tag/mappers/TagUpdateInputMapper.java | 3 ++ .../owner/AddOwnersResolverTest.java | 20 ++++++------- .../owner/BatchAddOwnersResolverTest.java | 30 +++++++++---------- 11 files changed, 49 insertions(+), 37 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java index d07bc25a552b4..d89f52bb7a03e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java @@ -74,7 +74,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws String domainUrn = _entityClient.ingestProposal(proposal, context.getAuthentication()); OwnershipType ownershipType = OwnershipType.TECHNICAL_OWNER; - if (!_entityService.exists(UrnUtils.getUrn(mapOwnershipTypeToEntity(ownershipType)))) { + if (!_entityService.exists(UrnUtils.getUrn(mapOwnershipTypeToEntity(ownershipType.name())))) { log.warn("Technical owner does not exist, defaulting to None ownership."); ownershipType = OwnershipType.NONE; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolver.java index effee84ab5d0b..ee5b896043d62 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolver.java @@ -69,7 +69,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws String glossaryNodeUrn = _entityClient.ingestProposal(proposal, context.getAuthentication()); OwnershipType ownershipType = OwnershipType.TECHNICAL_OWNER; - if (!_entityService.exists(UrnUtils.getUrn(mapOwnershipTypeToEntity(ownershipType)))) { + if (!_entityService.exists(UrnUtils.getUrn(mapOwnershipTypeToEntity(ownershipType.name())))) { log.warn("Technical owner does not exist, defaulting to None ownership."); ownershipType = OwnershipType.NONE; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryTermResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryTermResolver.java index 13f4de3a4537f..6db79e5c347ea 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryTermResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryTermResolver.java @@ -68,7 +68,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws String glossaryTermUrn = _entityClient.ingestProposal(proposal, context.getAuthentication()); OwnershipType ownershipType = OwnershipType.TECHNICAL_OWNER; - if (!_entityService.exists(UrnUtils.getUrn(mapOwnershipTypeToEntity(ownershipType)))) { + if (!_entityService.exists(UrnUtils.getUrn(mapOwnershipTypeToEntity(ownershipType.name())))) { log.warn("Technical owner does not exist, defaulting to None ownership."); ownershipType = OwnershipType.NONE; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddOwnerResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddOwnerResolver.java index c9b8a626fc25d..5ca7007d98e43 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddOwnerResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddOwnerResolver.java @@ -36,7 +36,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw Urn ownerUrn = Urn.createFromString(input.getOwnerUrn()); OwnerEntityType ownerEntityType = input.getOwnerEntityType(); OwnershipType type = input.getType() == null ? OwnershipType.NONE : input.getType(); - String ownershipUrn = input.getOwnershipTypeUrn() == null ? mapOwnershipTypeToEntity(type) : input.getOwnershipTypeUrn(); + String ownershipUrn = input.getOwnershipTypeUrn() == null ? mapOwnershipTypeToEntity(type.name()) : input.getOwnershipTypeUrn(); Urn targetUrn = Urn.createFromString(input.getResourceUrn()); if (!OwnerUtils.isAuthorizedToUpdateOwners(environment.getContext(), targetUrn)) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java index fb9ca78d5d6f7..a062016ed5d7e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java @@ -116,7 +116,8 @@ private static void addOwner(Ownership ownershipAspect, Urn ownerUrn, OwnershipT } // Fall back to mapping deprecated type to the new ownership entity, if it matches remove - return mapOwnershipTypeToEntity(OwnershipType.valueOf(owner.getType().toString())).equals(ownershipUrn.toString()); + return mapOwnershipTypeToEntity(OwnershipType.valueOf(owner.getType().toString()).name()) + .equals(ownershipUrn.toString()); }); Owner newOwner = new Owner(); @@ -158,7 +159,7 @@ private static void removeOwnersIfExists(Ownership ownership, List ownerUrn } // Fall back to mapping deprecated type to the new ownership entity, if it matches remove - return mapOwnershipTypeToEntity(OwnershipType.valueOf(owner.getType().toString())) + return mapOwnershipTypeToEntity(OwnershipType.valueOf(owner.getType().toString()).name()) .equals(maybeOwnershipTypeUrn.get().toString()); }); } else { @@ -283,7 +284,7 @@ public static void addCreatorAsOwner( EntityService entityService) { try { Urn actorUrn = CorpuserUrn.createFromString(context.getActorUrn()); - String ownershipTypeUrn = mapOwnershipTypeToEntity(ownershipType); + String ownershipTypeUrn = mapOwnershipTypeToEntity(ownershipType.name()); if (!entityService.exists(UrnUtils.getUrn(ownershipTypeUrn))) { throw new RuntimeException(String.format("Unknown ownership type urn %s", ownershipTypeUrn)); @@ -300,8 +301,8 @@ public static void addCreatorAsOwner( } } - public static String mapOwnershipTypeToEntity(OwnershipType type) { - final String typeName = SYSTEM_ID + type.name().toLowerCase(); + public static String mapOwnershipTypeToEntity(String type) { + final String typeName = SYSTEM_ID + type.toLowerCase(); return Urn.createFromTuple(Constants.OWNERSHIP_TYPE_ENTITY_NAME, typeName).toString(); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolver.java index e253b67d3c739..a0568937d2651 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolver.java @@ -73,7 +73,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws String tagUrn = _entityClient.ingestProposal(proposal, context.getAuthentication()); OwnershipType ownershipType = OwnershipType.TECHNICAL_OWNER; - if (!_entityService.exists(UrnUtils.getUrn(mapOwnershipTypeToEntity(ownershipType)))) { + if (!_entityService.exists(UrnUtils.getUrn(mapOwnershipTypeToEntity(ownershipType.name())))) { log.warn("Technical owner does not exist, defaulting to None ownership."); ownershipType = OwnershipType.NONE; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/OwnerMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/OwnerMapper.java index f7c262db8535f..181bdc176fb94 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/OwnerMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/OwnerMapper.java @@ -33,7 +33,7 @@ public Owner apply(@Nonnull final com.linkedin.common.Owner owner, @Nonnull fina if (owner.getTypeUrn() == null) { OwnershipType ownershipType = OwnershipType.valueOf(owner.getType().toString()); - owner.setTypeUrn(UrnUtils.getUrn(mapOwnershipTypeToEntity(ownershipType))); + owner.setTypeUrn(UrnUtils.getUrn(mapOwnershipTypeToEntity(ownershipType.name()))); } if (owner.getTypeUrn() != null) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/OwnerUpdateMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/OwnerUpdateMapper.java index 4b740e0450a8f..aac0067974a69 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/OwnerUpdateMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/OwnerUpdateMapper.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql.types.common.mappers; +import com.linkedin.common.urn.UrnUtils; import javax.annotation.Nonnull; import com.linkedin.common.Owner; @@ -34,7 +35,14 @@ public Owner apply(@Nonnull final OwnerUpdate input) { } catch (URISyntaxException e) { e.printStackTrace(); } - owner.setType(OwnershipType.valueOf(input.getType().toString())); + if (input.getOwnershipTypeUrn() != null) { + owner.setTypeUrn(UrnUtils.getUrn(input.getOwnershipTypeUrn())); + } else if (input.getType() != null) { + owner.setType(OwnershipType.valueOf(input.getType().toString())); + } else { + throw new RuntimeException("Ownership type not specified. Please define the ownership type urn."); + } + owner.setSource(new OwnershipSource().setType(OwnershipSourceType.SERVICE)); return owner; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/TagUpdateInputMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/TagUpdateInputMapper.java index b666bf5c60ed5..505dd0d36954b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/TagUpdateInputMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/TagUpdateInputMapper.java @@ -8,6 +8,7 @@ import com.linkedin.common.OwnershipSourceType; import com.linkedin.common.OwnershipType; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.SetMode; import com.linkedin.datahub.graphql.generated.TagUpdateInput; import com.linkedin.datahub.graphql.types.common.mappers.util.UpdateMappingHelper; @@ -18,6 +19,7 @@ import java.util.Collection; import javax.annotation.Nonnull; +import static com.linkedin.datahub.graphql.resolvers.mutate.util.OwnerUtils.*; import static com.linkedin.metadata.Constants.*; @@ -47,6 +49,7 @@ public Collection apply( final Owner owner = new Owner(); owner.setOwner(actor); owner.setType(OwnershipType.NONE); + owner.setTypeUrn(UrnUtils.getUrn(mapOwnershipTypeToEntity(OwnershipType.NONE.name()))); owner.setSource(new OwnershipSource().setType(OwnershipSourceType.SERVICE)); ownership.setOwners(new OwnerArray(owner)); ownership.setLastModified(auditStamp); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/AddOwnersResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/AddOwnersResolverTest.java index 5d180e83475e6..9bdb3c1db2b0a 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/AddOwnersResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/AddOwnersResolverTest.java @@ -43,7 +43,7 @@ public void testGetSuccessNoExistingOwners() throws Exception { Mockito.when(mockService.exists(Urn.createFromString(TEST_OWNER_2_URN))).thenReturn(true); Mockito.when(mockService.exists(Urn.createFromString( - OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.TECHNICAL_OWNER)))) + OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.TECHNICAL_OWNER.name())))) .thenReturn(true); AddOwnersResolver resolver = new AddOwnersResolver(mockService); @@ -53,9 +53,9 @@ public void testGetSuccessNoExistingOwners() throws Exception { DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); AddOwnersInput input = new AddOwnersInput(ImmutableList.of( new OwnerInput(TEST_OWNER_1_URN, OwnerEntityType.CORP_USER, OwnershipType.TECHNICAL_OWNER, - OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER)), + OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER.name())), new OwnerInput(TEST_OWNER_2_URN, OwnerEntityType.CORP_USER, OwnershipType.TECHNICAL_OWNER, - OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER)) + OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER.name())) ), TEST_ENTITY_URN); Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); Mockito.when(mockEnv.getContext()).thenReturn(mockContext); @@ -88,7 +88,7 @@ public void testGetSuccessExistingOwners() throws Exception { Mockito.when(mockService.exists(Urn.createFromString(TEST_OWNER_2_URN))).thenReturn(true); Mockito.when(mockService.exists(Urn.createFromString( - OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.TECHNICAL_OWNER)))) + OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.TECHNICAL_OWNER.name())))) .thenReturn(true); AddOwnersResolver resolver = new AddOwnersResolver(mockService); @@ -98,9 +98,9 @@ public void testGetSuccessExistingOwners() throws Exception { DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); AddOwnersInput input = new AddOwnersInput(ImmutableList.of( new OwnerInput(TEST_OWNER_1_URN, OwnerEntityType.CORP_USER, OwnershipType.TECHNICAL_OWNER, - OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER)), + OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER.name())), new OwnerInput(TEST_OWNER_2_URN, OwnerEntityType.CORP_USER, OwnershipType.TECHNICAL_OWNER, - OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER)) + OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER.name())) ), TEST_ENTITY_URN); Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); Mockito.when(mockEnv.getContext()).thenReturn(mockContext); @@ -138,7 +138,7 @@ public void testGetFailureOwnerDoesNotExist() throws Exception { DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); AddOwnersInput input = new AddOwnersInput(ImmutableList.of( new OwnerInput(TEST_OWNER_1_URN, OwnerEntityType.CORP_USER, OwnershipType.TECHNICAL_OWNER, - OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER))), TEST_ENTITY_URN); + OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER.name()))), TEST_ENTITY_URN); Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); Mockito.when(mockEnv.getContext()).thenReturn(mockContext); @@ -166,7 +166,7 @@ public void testGetFailureResourceDoesNotExist() throws Exception { DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); AddOwnersInput input = new AddOwnersInput(ImmutableList.of( new OwnerInput(TEST_OWNER_1_URN, OwnerEntityType.CORP_USER, OwnershipType.TECHNICAL_OWNER, - OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER))), TEST_ENTITY_URN); + OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER.name()))), TEST_ENTITY_URN); Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); Mockito.when(mockEnv.getContext()).thenReturn(mockContext); @@ -184,7 +184,7 @@ public void testGetUnauthorized() throws Exception { DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); AddOwnersInput input = new AddOwnersInput(ImmutableList.of( new OwnerInput(TEST_OWNER_1_URN, OwnerEntityType.CORP_USER, OwnershipType.TECHNICAL_OWNER, - OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER))), TEST_ENTITY_URN); + OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER.name()))), TEST_ENTITY_URN); Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); QueryContext mockContext = getMockDenyContext(); Mockito.when(mockEnv.getContext()).thenReturn(mockContext); @@ -208,7 +208,7 @@ public void testGetEntityClientException() throws Exception { QueryContext mockContext = getMockAllowContext(); AddOwnersInput input = new AddOwnersInput(ImmutableList.of( new OwnerInput(TEST_OWNER_1_URN, OwnerEntityType.CORP_USER, OwnershipType.TECHNICAL_OWNER, - OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER))), TEST_ENTITY_URN); + OwnerUtils.mapOwnershipTypeToEntity(OwnershipType.TECHNICAL_OWNER.name()))), TEST_ENTITY_URN); Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); Mockito.when(mockEnv.getContext()).thenReturn(mockContext); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/BatchAddOwnersResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/BatchAddOwnersResolverTest.java index 5fc79f275caa1..e38eb9eb677c2 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/BatchAddOwnersResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/BatchAddOwnersResolverTest.java @@ -56,7 +56,7 @@ public void testGetSuccessNoExistingOwners() throws Exception { Mockito.when(mockService.exists(Urn.createFromString(TEST_OWNER_URN_2))).thenReturn(true); Mockito.when(mockService.exists(Urn.createFromString( - OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER)))) + OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER.name())))) .thenReturn(true); BatchAddOwnersResolver resolver = new BatchAddOwnersResolver(mockService); @@ -68,12 +68,12 @@ public void testGetSuccessNoExistingOwners() throws Exception { TEST_OWNER_URN_1, OwnerEntityType.CORP_USER, com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER, - OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER)), + OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER.name())), new OwnerInput( TEST_OWNER_URN_2, OwnerEntityType.CORP_USER, com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER, - OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER))), + OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER.name()))), null, ImmutableList.of( new ResourceRefInput(TEST_ENTITY_URN_1, null, null), @@ -119,11 +119,11 @@ public void testGetSuccessExistingOwners() throws Exception { Mockito.when(mockService.exists(Urn.createFromString(TEST_OWNER_URN_2))).thenReturn(true); Mockito.when(mockService.exists(Urn.createFromString( - OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.TECHNICAL_OWNER)))) + OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.TECHNICAL_OWNER.name())))) .thenReturn(true); Mockito.when(mockService.exists(Urn.createFromString( - OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER)))) + OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER.name())))) .thenReturn(true); BatchAddOwnersResolver resolver = new BatchAddOwnersResolver(mockService); @@ -136,12 +136,12 @@ public void testGetSuccessExistingOwners() throws Exception { TEST_OWNER_URN_1, OwnerEntityType.CORP_USER, com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER, - OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER)), + OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER.name())), new OwnerInput( TEST_OWNER_URN_2, OwnerEntityType.CORP_USER, com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER, - OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER))), + OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER.name()))), null, ImmutableList.of( new ResourceRefInput(TEST_ENTITY_URN_1, null, null), @@ -183,12 +183,12 @@ public void testGetFailureOwnerDoesNotExist() throws Exception { TEST_OWNER_URN_1, OwnerEntityType.CORP_USER, com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER, - OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER)), + OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER.name())), new OwnerInput( TEST_OWNER_URN_2, OwnerEntityType.CORP_USER, com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER, - OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER))), + OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER.name()))), null, ImmutableList.of( new ResourceRefInput(TEST_ENTITY_URN_1, null, null), @@ -228,12 +228,12 @@ public void testGetFailureResourceDoesNotExist() throws Exception { TEST_OWNER_URN_1, OwnerEntityType.CORP_USER, com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER, - OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER)), + OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER.name())), new OwnerInput( TEST_OWNER_URN_2, OwnerEntityType.CORP_USER, com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER, - OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER))), + OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER.name()))), null, ImmutableList.of( new ResourceRefInput(TEST_ENTITY_URN_1, null, null), @@ -257,12 +257,12 @@ public void testGetUnauthorized() throws Exception { TEST_OWNER_URN_1, OwnerEntityType.CORP_USER, com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER, - OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER)), + OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER.name())), new OwnerInput( TEST_OWNER_URN_2, OwnerEntityType.CORP_USER, com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER, - OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER))), + OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER.name()))), null, ImmutableList.of( new ResourceRefInput(TEST_ENTITY_URN_1, null, null), @@ -292,12 +292,12 @@ public void testGetEntityClientException() throws Exception { TEST_OWNER_URN_1, OwnerEntityType.CORP_USER, com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER, - OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER)), + OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER.name())), new OwnerInput( TEST_OWNER_URN_2, OwnerEntityType.CORP_USER, com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER, - OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER))), + OwnerUtils.mapOwnershipTypeToEntity(com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER.name()))), null, ImmutableList.of( new ResourceRefInput(TEST_ENTITY_URN_1, null, null), From e5f9ee178c3139a28b1cda784e95acd10bdb3ef1 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 13 Jun 2023 12:34:06 -0700 Subject: [PATCH 007/222] fix(cli/quickstart): handle docker hangs gracefully (#8211) Co-authored-by: Pedro Silva --- docker/docker-compose-without-neo4j.yml | 18 +++++------ docker/docker-compose.override.yml | 4 +-- docker/docker-compose.yml | 18 +++++------ .../docker-compose-m1.quickstart.yml | 22 ++++++------- ...er-compose-without-neo4j-m1.quickstart.yml | 18 +++++------ ...ocker-compose-without-neo4j.quickstart.yml | 18 +++++------ .../quickstart/docker-compose.quickstart.yml | 22 ++++++------- .../src/datahub/cli/docker_cli.py | 32 ++++++++++++++++--- 8 files changed, 87 insertions(+), 65 deletions(-) diff --git a/docker/docker-compose-without-neo4j.yml b/docker/docker-compose-without-neo4j.yml index 87ec6105d8298..15d327847936a 100644 --- a/docker/docker-compose-without-neo4j.yml +++ b/docker/docker-compose-without-neo4j.yml @@ -42,9 +42,9 @@ services: env_file: datahub-gms/env/docker-without-neo4j.env healthcheck: test: curl -sS --fail http://datahub-gms:${DATAHUB_MAPPED_GMS_PORT:-8080}/health - start_period: 20s + start_period: 90s interval: 1s - retries: 20 + retries: 3 timeout: 5s depends_on: datahub-upgrade: @@ -116,9 +116,9 @@ services: memory: 1G healthcheck: test: curl -sS --fail http://elasticsearch:$${DATAHUB_MAPPED_ELASTIC_PORT:-9200}/_cluster/health?wait_for_status=yellow&timeout=0s - start_period: 5s + start_period: 20s interval: 1s - retries: 5 + retries: 3 timeout: 5s volumes: - esdata:/usr/share/elasticsearch/data @@ -131,9 +131,9 @@ services: env_file: schema-registry/env/docker.env healthcheck: test: nc -z schema-registry ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081} - start_period: 5s + start_period: 30s interval: 1s - retries: 5 + retries: 3 timeout: 5s depends_on: broker: @@ -147,7 +147,7 @@ services: env_file: broker/env/docker.env healthcheck: test: nc -z broker $${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092} - start_period: 5s + start_period: 30s interval: 1s retries: 5 timeout: 5s @@ -165,9 +165,9 @@ services: env_file: zookeeper/env/docker.env healthcheck: test: echo srvr | nc zookeeper $${DATAHUB_MAPPED_ZK_PORT:-2181} - start_period: 2s + start_period: 10s interval: 5s - retries: 5 + retries: 3 timeout: 5s volumes: - zkdata:/var/lib/zookeeper diff --git a/docker/docker-compose.override.yml b/docker/docker-compose.override.yml index 29665036ee39b..225aa01fa4e4f 100644 --- a/docker/docker-compose.override.yml +++ b/docker/docker-compose.override.yml @@ -36,9 +36,9 @@ services: restart: on-failure healthcheck: test: mysqladmin ping -h mysql -u $$MYSQL_USER --password=$$MYSQL_PASSWORD - start_period: 2s + start_period: 10s interval: 1s - retries: 5 + retries: 3 timeout: 5s volumes: - ./mysql/init.sql:/docker-entrypoint-initdb.d/init.sql diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b2c55fafdb26b..f242cb730a68f 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -40,9 +40,9 @@ services: dockerfile: docker/datahub-gms/Dockerfile healthcheck: test: curl -sS --fail http://datahub-gms:${DATAHUB_MAPPED_GMS_PORT:-8080}/health - start_period: 20s + start_period: 90s interval: 1s - retries: 20 + retries: 3 timeout: 5s depends_on: datahub-upgrade: @@ -119,9 +119,9 @@ services: memory: 1G healthcheck: test: curl -sS --fail http://elasticsearch:$${DATAHUB_MAPPED_ELASTIC_PORT:-9200}/_cluster/health?wait_for_status=yellow&timeout=0s - start_period: 5s + start_period: 20s interval: 1s - retries: 5 + retries: 3 timeout: 5s volumes: - esdata:/usr/share/elasticsearch/data @@ -150,9 +150,9 @@ services: env_file: schema-registry/env/docker.env healthcheck: test: nc -z schema-registry ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081} - start_period: 5s + start_period: 30s interval: 1s - retries: 5 + retries: 3 timeout: 5s depends_on: broker: @@ -166,7 +166,7 @@ services: env_file: broker/env/docker.env healthcheck: test: nc -z broker $${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092} - start_period: 5s + start_period: 30s interval: 1s retries: 5 timeout: 5s @@ -184,9 +184,9 @@ services: env_file: zookeeper/env/docker.env healthcheck: test: echo srvr | nc zookeeper $${DATAHUB_MAPPED_ZK_PORT:-2181} - start_period: 2s + start_period: 10s interval: 5s - retries: 5 + retries: 3 timeout: 5s volumes: - zkdata:/var/lib/zookeeper diff --git a/docker/quickstart/docker-compose-m1.quickstart.yml b/docker/quickstart/docker-compose-m1.quickstart.yml index ab54123fbe646..5ee039b1ceb42 100644 --- a/docker/quickstart/docker-compose-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-m1.quickstart.yml @@ -19,7 +19,7 @@ services: healthcheck: interval: 1s retries: 5 - start_period: 5s + start_period: 30s test: nc -z broker $${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092} timeout: 5s hostname: broker @@ -105,8 +105,8 @@ services: - METADATA_SERVICE_AUTH_ENABLED=false healthcheck: interval: 1s - retries: 20 - start_period: 20s + retries: 3 + start_period: 90s test: curl -sS --fail http://datahub-gms:${DATAHUB_MAPPED_GMS_PORT:-8080}/health timeout: 5s hostname: datahub-gms @@ -162,8 +162,8 @@ services: - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true healthcheck: interval: 1s - retries: 5 - start_period: 5s + retries: 3 + start_period: 20s test: curl -sS --fail http://elasticsearch:$${DATAHUB_MAPPED_ELASTIC_PORT:-9200}/_cluster/health?wait_for_status=yellow&timeout=0s timeout: 5s hostname: elasticsearch @@ -211,8 +211,8 @@ services: - MYSQL_ROOT_PASSWORD=datahub healthcheck: interval: 1s - retries: 5 - start_period: 2s + retries: 3 + start_period: 10s test: mysqladmin ping -h mysql -u $$MYSQL_USER --password=$$MYSQL_PASSWORD timeout: 5s hostname: mysql @@ -268,8 +268,8 @@ services: - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092 healthcheck: interval: 1s - retries: 5 - start_period: 5s + retries: 3 + start_period: 30s test: nc -z schema-registry ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081} timeout: 5s hostname: schema-registry @@ -283,8 +283,8 @@ services: - ZOOKEEPER_TICK_TIME=2000 healthcheck: interval: 5s - retries: 5 - start_period: 2s + retries: 3 + start_period: 10s test: echo srvr | nc zookeeper $${DATAHUB_MAPPED_ZK_PORT:-2181} timeout: 5s hostname: zookeeper diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index 7233702fdef24..2ad4f33c6dd99 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -19,7 +19,7 @@ services: healthcheck: interval: 1s retries: 5 - start_period: 5s + start_period: 30s test: nc -z broker $${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092} timeout: 5s hostname: broker @@ -100,8 +100,8 @@ services: - UI_INGESTION_ENABLED=true healthcheck: interval: 1s - retries: 20 - start_period: 20s + retries: 3 + start_period: 90s test: curl -sS --fail http://datahub-gms:${DATAHUB_MAPPED_GMS_PORT:-8080}/health timeout: 5s hostname: datahub-gms @@ -155,8 +155,8 @@ services: - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true healthcheck: interval: 1s - retries: 5 - start_period: 5s + retries: 3 + start_period: 20s test: curl -sS --fail http://elasticsearch:$${DATAHUB_MAPPED_ELASTIC_PORT:-9200}/_cluster/health?wait_for_status=yellow&timeout=0s timeout: 5s hostname: elasticsearch @@ -242,8 +242,8 @@ services: - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092 healthcheck: interval: 1s - retries: 5 - start_period: 5s + retries: 3 + start_period: 30s test: nc -z schema-registry ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081} timeout: 5s hostname: schema-registry @@ -257,8 +257,8 @@ services: - ZOOKEEPER_TICK_TIME=2000 healthcheck: interval: 5s - retries: 5 - start_period: 2s + retries: 3 + start_period: 10s test: echo srvr | nc zookeeper $${DATAHUB_MAPPED_ZK_PORT:-2181} timeout: 5s hostname: zookeeper diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index 49dab99e383a8..f86d9756ef855 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -19,7 +19,7 @@ services: healthcheck: interval: 1s retries: 5 - start_period: 5s + start_period: 30s test: nc -z broker $${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092} timeout: 5s hostname: broker @@ -100,8 +100,8 @@ services: - UI_INGESTION_ENABLED=true healthcheck: interval: 1s - retries: 20 - start_period: 20s + retries: 3 + start_period: 90s test: curl -sS --fail http://datahub-gms:${DATAHUB_MAPPED_GMS_PORT:-8080}/health timeout: 5s hostname: datahub-gms @@ -155,8 +155,8 @@ services: - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true healthcheck: interval: 1s - retries: 5 - start_period: 5s + retries: 3 + start_period: 20s test: curl -sS --fail http://elasticsearch:$${DATAHUB_MAPPED_ELASTIC_PORT:-9200}/_cluster/health?wait_for_status=yellow&timeout=0s timeout: 5s hostname: elasticsearch @@ -242,8 +242,8 @@ services: - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092 healthcheck: interval: 1s - retries: 5 - start_period: 5s + retries: 3 + start_period: 30s test: nc -z schema-registry ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081} timeout: 5s hostname: schema-registry @@ -257,8 +257,8 @@ services: - ZOOKEEPER_TICK_TIME=2000 healthcheck: interval: 5s - retries: 5 - start_period: 2s + retries: 3 + start_period: 10s test: echo srvr | nc zookeeper $${DATAHUB_MAPPED_ZK_PORT:-2181} timeout: 5s hostname: zookeeper diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index 379b6bd1edfa3..2587a76433cdb 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -19,7 +19,7 @@ services: healthcheck: interval: 1s retries: 5 - start_period: 5s + start_period: 30s test: nc -z broker $${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092} timeout: 5s hostname: broker @@ -105,8 +105,8 @@ services: - METADATA_SERVICE_AUTH_ENABLED=false healthcheck: interval: 1s - retries: 20 - start_period: 20s + retries: 3 + start_period: 90s test: curl -sS --fail http://datahub-gms:${DATAHUB_MAPPED_GMS_PORT:-8080}/health timeout: 5s hostname: datahub-gms @@ -162,8 +162,8 @@ services: - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true healthcheck: interval: 1s - retries: 5 - start_period: 5s + retries: 3 + start_period: 20s test: curl -sS --fail http://elasticsearch:$${DATAHUB_MAPPED_ELASTIC_PORT:-9200}/_cluster/health?wait_for_status=yellow&timeout=0s timeout: 5s hostname: elasticsearch @@ -211,8 +211,8 @@ services: - MYSQL_ROOT_PASSWORD=datahub healthcheck: interval: 1s - retries: 5 - start_period: 2s + retries: 3 + start_period: 10s test: mysqladmin ping -h mysql -u $$MYSQL_USER --password=$$MYSQL_PASSWORD timeout: 5s hostname: mysql @@ -268,8 +268,8 @@ services: - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092 healthcheck: interval: 1s - retries: 5 - start_period: 5s + retries: 3 + start_period: 30s test: nc -z schema-registry ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081} timeout: 5s hostname: schema-registry @@ -283,8 +283,8 @@ services: - ZOOKEEPER_TICK_TIME=2000 healthcheck: interval: 5s - retries: 5 - start_period: 2s + retries: 3 + start_period: 10s test: echo srvr | nc zookeeper $${DATAHUB_MAPPED_ZK_PORT:-2181} timeout: 5s hostname: zookeeper diff --git a/metadata-ingestion/src/datahub/cli/docker_cli.py b/metadata-ingestion/src/datahub/cli/docker_cli.py index 44dfab36951e5..e2b7b2a2e1ff4 100644 --- a/metadata-ingestion/src/datahub/cli/docker_cli.py +++ b/metadata-ingestion/src/datahub/cli/docker_cli.py @@ -34,6 +34,7 @@ from datahub.ingestion.run.pipeline import Pipeline from datahub.telemetry import telemetry from datahub.upgrade import upgrade +from datahub.utilities.perf_timer import PerfTimer from datahub.utilities.sample_data import BOOTSTRAP_MCES_FILE, download_sample_data logger = logging.getLogger(__name__) @@ -62,6 +63,7 @@ _QUICKSTART_MAX_WAIT_TIME = datetime.timedelta(minutes=10) +_QUICKSTART_UP_TIMEOUT = datetime.timedelta(seconds=100) _QUICKSTART_STATUS_CHECK_INTERVAL = datetime.timedelta(seconds=2) @@ -702,14 +704,28 @@ def quickstart( # As such, we'll only use the quiet flag if we're in an interactive environment. # If we're in quiet mode, then we'll show a spinner instead. quiet = not sys.stderr.isatty() - with click_spinner.spinner(disable=not quiet): + with PerfTimer() as timer, click_spinner.spinner(disable=not quiet): subprocess.run( [*base_command, "pull", *(("-q",) if quiet else ())], check=True, env=_docker_subprocess_env(), ) + + telemetry.telemetry_instance.ping( + "quickstart-image-pull", + { + "status": "success", + "duration": timer.elapsed_seconds(), + }, + ) click.secho("Finished pulling docker images!") except subprocess.CalledProcessError: + telemetry.telemetry_instance.ping( + "quickstart-image-pull", + { + "status": "failure", + }, + ) click.secho( "Error while pulling images. Going to attempt to move on to docker compose up assuming the images have " "been built locally", @@ -736,12 +752,18 @@ def quickstart( if up_attempts == 0 or (status and status.needs_up()): if up_attempts > 0: click.echo() - subprocess.run( - base_command + ["up", "-d", "--remove-orphans"], - env=_docker_subprocess_env(), - ) up_attempts += 1 + logger.debug(f"Executing docker compose up command, attempt #{up_attempts}") + try: + subprocess.run( + base_command + ["up", "-d", "--remove-orphans"], + env=_docker_subprocess_env(), + timeout=_QUICKSTART_UP_TIMEOUT.total_seconds(), + ) + except subprocess.TimeoutExpired: + logger.debug("docker compose up timed out, will retry") + # Check docker health every few seconds. status = check_docker_quickstart() if status.is_ok(): From 1528935d40c219ea5d2db28470f6c96290a7670a Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 13 Jun 2023 23:28:03 -0700 Subject: [PATCH 008/222] fix(cli): make quickstart robust to docker race conditions (#8233) --- metadata-ingestion/src/datahub/cli/docker_check.py | 3 +++ metadata-ingestion/src/datahub/telemetry/telemetry.py | 8 +++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/cli/docker_check.py b/metadata-ingestion/src/datahub/cli/docker_check.py index d6114a341654c..97b88cbc8b8eb 100644 --- a/metadata-ingestion/src/datahub/cli/docker_check.py +++ b/metadata-ingestion/src/datahub/cli/docker_check.py @@ -180,6 +180,9 @@ def check_docker_quickstart() -> QuickstartStatus: containers = client.containers.list( all=True, filters=DATAHUB_COMPOSE_PROJECT_FILTER, + # We can get race conditions between docker running up / recreating + # containers and our status checks. + ignore_removed=True, ) if len(containers) == 0: return QuickstartStatus([]) diff --git a/metadata-ingestion/src/datahub/telemetry/telemetry.py b/metadata-ingestion/src/datahub/telemetry/telemetry.py index 35c799444c4d5..06161bd1e94f9 100644 --- a/metadata-ingestion/src/datahub/telemetry/telemetry.py +++ b/metadata-ingestion/src/datahub/telemetry/telemetry.py @@ -15,6 +15,7 @@ from datahub.cli.cli_utils import DATAHUB_ROOT_FOLDER, get_boolean_env_variable from datahub.configuration.common import ExceptionWithProps from datahub.ingestion.graph.client import DataHubGraph +from datahub.utilities.perf_timer import PerfTimer logger = logging.getLogger(__name__) @@ -331,7 +332,12 @@ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _T: {**call_props, "status": "start"}, ) try: - res = func(*args, **kwargs) + try: + with PerfTimer() as timer: + res = func(*args, **kwargs) + finally: + call_props["duration"] = timer.elapsed_seconds() + telemetry_instance.ping( "function-call", {**call_props, "status": "completed"}, From 5fc6601d8d0fd5c1099c49ced9f846865b2946c7 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Wed, 14 Jun 2023 14:01:45 +0530 Subject: [PATCH 009/222] fix(search): tag/term should filter for both entity and field level (#7881) --- .../renderer/component/GlossaryTermSearchList.tsx | 6 ++++++ .../recommendations/renderer/component/TagSearchList.tsx | 6 ++++++ .../src/app/search/SearchResultsRecommendations.tsx | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/datahub-web-react/src/app/recommendations/renderer/component/GlossaryTermSearchList.tsx b/datahub-web-react/src/app/recommendations/renderer/component/GlossaryTermSearchList.tsx index 4ccb2fc8eb023..7c3c6604ebe18 100644 --- a/datahub-web-react/src/app/recommendations/renderer/component/GlossaryTermSearchList.tsx +++ b/datahub-web-react/src/app/recommendations/renderer/component/GlossaryTermSearchList.tsx @@ -6,6 +6,7 @@ import { useHistory } from 'react-router-dom'; import { RecommendationContent, GlossaryTerm } from '../../../../types.generated'; import { navigateToSearchUrl } from '../../../search/utils/navigateToSearchUrl'; import { useEntityRegistry } from '../../../useEntityRegistry'; +import { UnionType } from '../../../search/utils/constants'; const TermSearchListContainer = styled.div` display: flex; @@ -55,8 +56,13 @@ export const GlossaryTermSearchList = ({ content, onClick }: Props) => { field: 'glossaryTerms', values: [term.urn], }, + { + field: 'fieldGlossaryTerms', + values: [term.urn], + }, ], history, + unionType: UnionType.OR, }); }; diff --git a/datahub-web-react/src/app/recommendations/renderer/component/TagSearchList.tsx b/datahub-web-react/src/app/recommendations/renderer/component/TagSearchList.tsx index c6f0c401d0af8..62dff17683427 100644 --- a/datahub-web-react/src/app/recommendations/renderer/component/TagSearchList.tsx +++ b/datahub-web-react/src/app/recommendations/renderer/component/TagSearchList.tsx @@ -4,6 +4,7 @@ import { useHistory } from 'react-router-dom'; import styled from 'styled-components'; import { EntityType, RecommendationContent, Tag } from '../../../../types.generated'; import { StyledTag } from '../../../entity/shared/components/styled/StyledTag'; +import { UnionType } from '../../../search/utils/constants'; import { navigateToSearchUrl } from '../../../search/utils/navigateToSearchUrl'; import { useEntityRegistry } from '../../../useEntityRegistry'; @@ -46,8 +47,13 @@ export const TagSearchList = ({ content, onClick }: Props) => { field: 'tags', values: [tag.urn], }, + { + field: 'fieldTags', + values: [tag.urn], + }, ], history, + unionType: UnionType.OR, }); }; diff --git a/datahub-web-react/src/app/search/SearchResultsRecommendations.tsx b/datahub-web-react/src/app/search/SearchResultsRecommendations.tsx index ea16a86483247..92e5817b07685 100644 --- a/datahub-web-react/src/app/search/SearchResultsRecommendations.tsx +++ b/datahub-web-react/src/app/search/SearchResultsRecommendations.tsx @@ -55,7 +55,7 @@ export const SearchResultsRecommendations = ({ userUrn, query, filters }: Props) return ( <> {recommendationModules && !!recommendationModules.length && ( - + More you may be interested in {recommendationModules && recommendationModules.map((module) => ( From 0c22bda2099399e8c32d52652b5486248b2cc468 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Wed, 14 Jun 2023 19:19:23 +0530 Subject: [PATCH 010/222] docs(tests): document test eval endpoint (#8227) --- docs-website/sidebars.js | 5 +++++ docs/api/restli/evaluate-tests.md | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 docs/api/restli/evaluate-tests.md diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index d38d77066cbb6..43537c57433b4 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -288,6 +288,11 @@ module.exports = { label: "Restore Indices", id: "docs/api/restli/restore-indices", }, + { + type: "doc", + label: "Evaluate Tests", + id: "docs/api/restli/evaluate-tests", + }, { type: "doc", label: "Aspect Versioning and Rest.li Modeling", diff --git a/docs/api/restli/evaluate-tests.md b/docs/api/restli/evaluate-tests.md new file mode 100644 index 0000000000000..3e4b6d73928a4 --- /dev/null +++ b/docs/api/restli/evaluate-tests.md @@ -0,0 +1,18 @@ +# Evaluate Tests Endpoint + + +You can do a HTTP POST request to `/gms/test?action=evaluate` endpoint with the `urn` as part of JSON Payload to run metadata tests for the particular URN. + +``` +curl --location --request POST 'https://DOMAIN.acryl.io/gms/test?action=evaluate' \ +--header 'Authorization: Bearer TOKEN' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "urn": "YOUR_URN" +}' +``` +w +The supported parameters are +- `urn` - Required URN string +- `shouldPush` - Optional Boolean - whether or not to push the results to persist them +- `testUrns` - Optional List of string - If you wish to get specific test URNs evaluated From c5cc53b99a15d2bda762497a009c99a2e265c088 Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Wed, 14 Jun 2023 12:50:21 -0400 Subject: [PATCH 011/222] feat(ingest/bigquery_v2): enable platform instance using project id (#8216) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Pertíñez Co-authored-by: Harshal Sheth --- .../ingestion/source/bigquery_v2/bigquery.py | 20 +- .../bigquery_v2/bigquery_mcp_golden.json | 267 ++++++++++++++++++ .../integration/bigquery_v2/test_bigquery.py | 80 ++++++ .../tests/unit/test_bigquery_source.py | 20 ++ 4 files changed, 385 insertions(+), 2 deletions(-) create mode 100644 metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json create mode 100644 metadata-ingestion/tests/integration/bigquery_v2/test_bigquery.py diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py index f6f7912397a3d..4daee3a59e2ed 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py @@ -13,6 +13,7 @@ from datahub.configuration.pattern_utils import is_schema_allowed from datahub.emitter.mce_builder import ( make_data_platform_urn, + make_dataplatform_instance_urn, make_dataset_urn, make_tag_urn, set_dataset_urn_to_lower, @@ -105,6 +106,7 @@ TimeType, ) from datahub.metadata.schema_classes import ( + DataPlatformInstanceClass, DatasetLineageTypeClass, GlobalTagsClass, TagAssociationClass, @@ -138,9 +140,9 @@ def cleanup(config: BigQueryV2Config) -> None: @platform_name("BigQuery", doc_order=1) @config_class(BigQueryV2Config) @support_status(SupportStatus.CERTIFIED) -@capability( +@capability( # DataPlatformAspect is set to project id, but not added to urns as project id is in the container path SourceCapability.PLATFORM_INSTANCE, - "Not supported since BigQuery project ids are globally unique", + "Platform instance is pre-set to the BigQuery project id", supported=False, ) @capability(SourceCapability.DOMAINS, "Supported via the `domain` config field") @@ -398,6 +400,17 @@ def test_connection(config_dict: dict) -> TestConnectionReport: ) return test_report + def get_dataplatform_instance_aspect( + self, dataset_urn: str, project_id: str + ) -> MetadataWorkUnit: + aspect = DataPlatformInstanceClass( + platform=make_data_platform_urn(self.platform), + instance=make_dataplatform_instance_urn(self.platform, project_id), + ) + return MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=aspect + ).as_workunit() + def gen_dataset_key(self, db_name: str, schema: str) -> PlatformKey: return BigQueryDatasetKey( project_id=db_name, @@ -987,6 +1000,9 @@ def gen_dataset_workunits( dataset_urn=dataset_urn, parent_container_key=self.gen_dataset_key(project_id, dataset_name), ) + yield self.get_dataplatform_instance_aspect( + dataset_urn=dataset_urn, project_id=project_id + ) subTypes = SubTypes(typeNames=sub_types) yield MetadataChangeProposalWrapper( diff --git a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json new file mode 100644 index 0000000000000..38d36871fbde8 --- /dev/null +++ b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json @@ -0,0 +1,267 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:068bd9323110994a40019fcf6cfc60d3", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "bigquery", + "env": "PROD", + "project_id": "project-id-1" + }, + "name": "project-id-1" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:068bd9323110994a40019fcf6cfc60d3", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:068bd9323110994a40019fcf6cfc60d3", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:bigquery" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:068bd9323110994a40019fcf6cfc60d3", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:8df46c5e3ded05a3122b0015822c0ef0", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "bigquery", + "env": "PROD", + "project_id": "project-id-1", + "dataset_id": "bigquery-dataset-1" + }, + "externalUrl": "https://console.cloud.google.com/bigquery?project=project-id-1&ws=!1m4!1m3!3m2!1sproject-id-1!2sbigquery-dataset-1", + "name": "bigquery-dataset-1" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:8df46c5e3ded05a3122b0015822c0ef0", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:8df46c5e3ded05a3122b0015822c0ef0", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:bigquery" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:8df46c5e3ded05a3122b0015822c0ef0", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:8df46c5e3ded05a3122b0015822c0ef0", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:068bd9323110994a40019fcf6cfc60d3" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "project-id-1.bigquery-dataset-1.table-1", + "platform": "urn:li:dataPlatform:bigquery", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "https://console.cloud.google.com/bigquery?project=project-id-1&ws=!1m5!1m4!4m3!1sproject-id-1!2sbigquery-dataset-1!3stable-1", + "name": "table-1", + "qualifiedName": "project-id-1.bigquery-dataset-1.table-1", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:8df46c5e3ded05a3122b0015822c0ef0" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:bigquery", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:bigquery,project-id-1)" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery.py b/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery.py new file mode 100644 index 0000000000000..3bda6c5cce84b --- /dev/null +++ b/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery.py @@ -0,0 +1,80 @@ +from typing import Any, Dict +from unittest.mock import patch + +from freezegun import freeze_time +from google.cloud.bigquery.table import TableListItem + +from datahub.ingestion.source.bigquery_v2.bigquery_schema import ( + BigqueryDataset, + BigqueryTable, +) +from tests.test_helpers import mce_helpers +from tests.test_helpers.state_helpers import run_and_get_pipeline + +FROZEN_TIME = "2022-02-03 07:00:00" + + +@freeze_time(FROZEN_TIME) +@patch( + "datahub.ingestion.source.bigquery_v2.bigquery_schema.BigQueryDataDictionary.get_tables_for_dataset" +) +@patch( + "datahub.ingestion.source.bigquery_v2.bigquery.BigqueryV2Source.get_core_table_details" +) +@patch( + "datahub.ingestion.source.bigquery_v2.bigquery_schema.BigQueryDataDictionary.get_datasets_for_project_id" +) +@patch("google.cloud.bigquery.Client") +def test_bigquery_v2_ingest( + client, + get_datasets_for_project_id, + get_core_table_details, + get_tables_for_dataset, + pytestconfig, + tmp_path, +): + test_resources_dir = pytestconfig.rootpath / "tests/integration/bigquery_v2" + mcp_golden_path = "{}/bigquery_mcp_golden.json".format(test_resources_dir) + mcp_output_path = "{}/{}".format(tmp_path, "bigquery_mcp_output.json") + + get_datasets_for_project_id.return_value = [ + BigqueryDataset(name="bigquery-dataset-1") + ] + + table_list_item = TableListItem( + {"tableReference": {"projectId": "", "datasetId": "", "tableId": ""}} + ) + table_name = "table-1" + get_core_table_details.return_value = {table_name: table_list_item} + + bigquery_table = BigqueryTable( + name=table_name, + comment=None, + created=None, + last_altered=None, + size_in_bytes=None, + rows_count=None, + ) + get_tables_for_dataset.return_value = iter([bigquery_table]) + + source_config_dict: Dict[str, Any] = { + "project_ids": ["project-id-1"], + "include_usage_statistics": False, + "include_table_lineage": False, + } + + pipeline_config_dict: Dict[str, Any] = { + "source": { + "type": "bigquery", + "config": source_config_dict, + }, + "sink": {"type": "file", "config": {"filename": mcp_output_path}}, + } + + run_and_get_pipeline(pipeline_config_dict) + + mce_helpers.check_golden_file( + pytestconfig, + output_path=mcp_output_path, + golden_path=mcp_golden_path, + ) diff --git a/metadata-ingestion/tests/unit/test_bigquery_source.py b/metadata-ingestion/tests/unit/test_bigquery_source.py index e753284add49c..49dc66b2322f9 100644 --- a/metadata-ingestion/tests/unit/test_bigquery_source.py +++ b/metadata-ingestion/tests/unit/test_bigquery_source.py @@ -128,6 +128,26 @@ def test_get_projects_with_project_ids_overrides_project_id_pattern(): ] +def test_get_dataplatform_instance_aspect_returns_project_id(): + project_id = "project_id" + expected_instance = ( + f"urn:li:dataPlatformInstance:(urn:li:dataPlatform:bigquery,{project_id})" + ) + + config = BigQueryV2Config.parse_obj({}) + source = BigqueryV2Source(config=config, ctx=PipelineContext(run_id="test")) + + data_platform_instance = source.get_dataplatform_instance_aspect( + "urn:li:test", project_id + ) + + metadata = data_platform_instance.get_metadata()["metadata"] + + assert data_platform_instance is not None + assert metadata.aspectName == "dataPlatformInstance" + assert metadata.aspect.instance == expected_instance + + @patch("google.cloud.bigquery.client.Client") def test_get_projects_with_single_project_id(client_mock): config = BigQueryV2Config.parse_obj({"project_id": "test-3"}) From 071ef4d111ea16d0dec5948654e51ece1edf8819 Mon Sep 17 00:00:00 2001 From: Joshua Eilers Date: Wed, 14 Jun 2023 10:54:19 -0700 Subject: [PATCH 012/222] feat(stats): make rowcount more human readable (#8232) --- .../entity/chart/shared/ChartStatsSummary.tsx | 14 ++++-- .../shared/DashboardStatsSummary.tsx | 14 ++++-- .../dataset/shared/DatasetStatsSummary.tsx | 34 +++++++++---- .../entity/dataset/shared/ExpandingStat.tsx | 48 +++++++++++++++++++ .../shared/components/styled/StatsSummary.tsx | 8 ++-- .../sidebar/Dataset/StatsSidebarSection.tsx | 4 +- .../Dataset/Stats/snapshot/TableStats.tsx | 5 +- .../search/autoComplete/AutoCompleteItem.tsx | 2 +- .../AutoCompleteTooltipContent.tsx | 2 +- .../src/utils/formatter/index.ts | 41 ++++++++++------ 10 files changed, 132 insertions(+), 40 deletions(-) create mode 100644 datahub-web-react/src/app/entity/dataset/shared/ExpandingStat.tsx diff --git a/datahub-web-react/src/app/entity/chart/shared/ChartStatsSummary.tsx b/datahub-web-react/src/app/entity/chart/shared/ChartStatsSummary.tsx index 29b58ba756fc5..566a9f9db5fc8 100644 --- a/datahub-web-react/src/app/entity/chart/shared/ChartStatsSummary.tsx +++ b/datahub-web-react/src/app/entity/chart/shared/ChartStatsSummary.tsx @@ -6,6 +6,8 @@ import { formatNumberWithoutAbbreviation } from '../../../shared/formatNumber'; import { ANTD_GRAY } from '../../shared/constants'; import { toLocalDateTimeString, toRelativeTimeString } from '../../../shared/time/timeUtils'; import { StatsSummary } from '../../shared/components/styled/StatsSummary'; +import { countFormatter, needsFormatting } from '../../../../utils/formatter'; +import ExpandingStat from '../../dataset/shared/ExpandingStat'; const StatText = styled.span` color: ${ANTD_GRAY[8]}; @@ -33,9 +35,15 @@ export const ChartStatsSummary = ({ }: Props) => { const statsViews = [ (!!chartCount && ( - - {chartCount} charts - + ( + + {isExpanded ? formatNumberWithoutAbbreviation(chartCount) : countFormatter(chartCount)}{' '} + charts + + )} + /> )) || undefined, (!!viewCount && ( diff --git a/datahub-web-react/src/app/entity/dashboard/shared/DashboardStatsSummary.tsx b/datahub-web-react/src/app/entity/dashboard/shared/DashboardStatsSummary.tsx index 510c94efde68e..e8fb4c16aca9c 100644 --- a/datahub-web-react/src/app/entity/dashboard/shared/DashboardStatsSummary.tsx +++ b/datahub-web-react/src/app/entity/dashboard/shared/DashboardStatsSummary.tsx @@ -6,6 +6,8 @@ import { formatNumberWithoutAbbreviation } from '../../../shared/formatNumber'; import { ANTD_GRAY } from '../../shared/constants'; import { toLocalDateTimeString, toRelativeTimeString } from '../../../shared/time/timeUtils'; import { StatsSummary } from '../../shared/components/styled/StatsSummary'; +import { countFormatter, needsFormatting } from '../../../../utils/formatter'; +import ExpandingStat from '../../dataset/shared/ExpandingStat'; const StatText = styled.span` color: ${ANTD_GRAY[8]}; @@ -33,9 +35,15 @@ export const DashboardStatsSummary = ({ }: Props) => { const statsViews = [ (!!chartCount && ( - - {chartCount} charts - + ( + + {isExpanded ? formatNumberWithoutAbbreviation(chartCount) : countFormatter(chartCount)}{' '} + charts + + )} + /> )) || undefined, (!!viewCount && ( diff --git a/datahub-web-react/src/app/entity/dataset/shared/DatasetStatsSummary.tsx b/datahub-web-react/src/app/entity/dataset/shared/DatasetStatsSummary.tsx index 0dcc06bc2a09d..14f550de25be7 100644 --- a/datahub-web-react/src/app/entity/dataset/shared/DatasetStatsSummary.tsx +++ b/datahub-web-react/src/app/entity/dataset/shared/DatasetStatsSummary.tsx @@ -7,6 +7,8 @@ import { ANTD_GRAY } from '../../shared/constants'; import { toLocalDateTimeString, toRelativeTimeString } from '../../../shared/time/timeUtils'; import { StatsSummary } from '../../shared/components/styled/StatsSummary'; import { FormattedBytesStat } from './FormattedBytesStat'; +import { countFormatter, needsFormatting } from '../../../../utils/formatter'; +import ExpandingStat from './ExpandingStat'; const StatText = styled.span<{ color: string }>` color: ${(props) => props.color}; @@ -25,6 +27,7 @@ type Props = { uniqueUserCountLast30Days?: number | null; lastUpdatedMs?: number | null; color?: string; + mode?: 'normal' | 'tooltip-content'; }; export const DatasetStatsSummary = ({ @@ -36,20 +39,33 @@ export const DatasetStatsSummary = ({ uniqueUserCountLast30Days, lastUpdatedMs, color, + mode = 'normal', }: Props) => { - const displayedColor = color !== undefined ? color : ANTD_GRAY[7]; + const isTooltipMode = mode === 'tooltip-content'; + const displayedColor = isTooltipMode ? '' : color ?? ANTD_GRAY[7]; const statsViews = [ !!rowCount && ( - - - {formatNumberWithoutAbbreviation(rowCount)} rows - {!!columnCount && ( - <> - , {formatNumberWithoutAbbreviation(columnCount)} columns - + ( + + + {isExpanded ? formatNumberWithoutAbbreviation(rowCount) : countFormatter(rowCount)} rows + {!!columnCount && ( + <> + ,{' '} + + {isExpanded + ? formatNumberWithoutAbbreviation(columnCount) + : countFormatter(columnCount)} + {' '} + columns + + )} + )} - + /> ), !!sizeInBytes && ( diff --git a/datahub-web-react/src/app/entity/dataset/shared/ExpandingStat.tsx b/datahub-web-react/src/app/entity/dataset/shared/ExpandingStat.tsx new file mode 100644 index 0000000000000..8101a696bf274 --- /dev/null +++ b/datahub-web-react/src/app/entity/dataset/shared/ExpandingStat.tsx @@ -0,0 +1,48 @@ +import React, { ReactNode, useEffect, useRef, useState } from 'react'; +import styled from 'styled-components'; + +const ExpandingStatContainer = styled.span<{ disabled: boolean; expanded: boolean; width: string }>` + overflow: hidden; + white-space: nowrap; + width: ${(props) => props.width}; + transition: width 250ms ease; +`; + +const ExpandingStat = ({ + disabled = false, + render, +}: { + disabled?: boolean; + render: (isExpanded: boolean) => ReactNode; +}) => { + const contentRef = useRef(null); + const [width, setWidth] = useState('inherit'); + const [isExpanded, setIsExpanded] = useState(false); + + useEffect(() => { + if (!contentRef.current) return; + setWidth(`${contentRef.current.offsetWidth}px`); + }, [isExpanded]); + + const onMouseEnter = () => { + if (!disabled) setIsExpanded(true); + }; + + const onMouseLeave = () => { + if (!disabled) setIsExpanded(false); + }; + + return ( + + {render(isExpanded)} + + ); +}; + +export default ExpandingStat; diff --git a/datahub-web-react/src/app/entity/shared/components/styled/StatsSummary.tsx b/datahub-web-react/src/app/entity/shared/components/styled/StatsSummary.tsx index 8146fa62bc219..a0fe5ef031bb7 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/StatsSummary.tsx +++ b/datahub-web-react/src/app/entity/shared/components/styled/StatsSummary.tsx @@ -8,15 +8,15 @@ type Props = { const StatsContainer = styled.div` margin-top: 8px; + display: flex; + align-items: center; `; const StatDivider = styled.div` - display: inline-block; padding-left: 10px; margin-right: 10px; border-right: 1px solid ${ANTD_GRAY[4]}; height: 21px; - vertical-align: text-top; `; export const StatsSummary = ({ stats }: Props) => { @@ -25,10 +25,10 @@ export const StatsSummary = ({ stats }: Props) => { {stats && stats.length > 0 && ( {stats.map((statView, index) => ( - + <> {statView} {index < stats.length - 1 && } - + ))} )} diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Dataset/StatsSidebarSection.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Dataset/StatsSidebarSection.tsx index a1c60fe00e9bb..dfde26412af8b 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Dataset/StatsSidebarSection.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Dataset/StatsSidebarSection.tsx @@ -8,7 +8,7 @@ import { ANTD_GRAY } from '../../../../constants'; import { useBaseEntity, useRouteToTab } from '../../../../EntityContext'; import { SidebarHeader } from '../SidebarHeader'; import { InfoItem } from '../../../../components/styled/InfoItem'; -import { countSeparator } from '../../../../../../../utils/formatter/index'; +import { formatNumberWithoutAbbreviation } from '../../../../../../shared/formatNumber'; const HeaderInfoBody = styled(Typography.Text)` font-size: 16px; @@ -83,7 +83,7 @@ export const SidebarStatsSection = () => { onClick={() => routeToTab({ tabName: 'Queries' })} width={INFO_ITEM_WIDTH_PX} > - {countSeparator(latestProfile?.rowCount)} + {formatNumberWithoutAbbreviation(latestProfile?.rowCount)} ) : null} {latestProfile?.columnCount ? ( diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/snapshot/TableStats.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/snapshot/TableStats.tsx index 62e5c7a62791b..eb39b9f4200b9 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/snapshot/TableStats.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/snapshot/TableStats.tsx @@ -4,8 +4,9 @@ import styled from 'styled-components'; import { CorpUser, Maybe, UserUsageCounts } from '../../../../../../../types.generated'; import { InfoItem } from '../../../../components/styled/InfoItem'; import { ANTD_GRAY } from '../../../../constants'; -import { countFormatter, countSeparator } from '../../../../../../../utils/formatter/index'; +import { countFormatter } from '../../../../../../../utils/formatter/index'; import { ExpandedActorGroup } from '../../../../components/styled/ExpandedActorGroup'; +import { formatNumberWithoutAbbreviation } from '../../../../../../shared/formatNumber'; type Props = { rowCount?: number; @@ -57,7 +58,7 @@ export default function TableStats({ {rowCount && ( - + {countFormatter(rowCount)} diff --git a/datahub-web-react/src/app/search/autoComplete/AutoCompleteItem.tsx b/datahub-web-react/src/app/search/autoComplete/AutoCompleteItem.tsx index 29cc549df0db2..c97d171b4c931 100644 --- a/datahub-web-react/src/app/search/autoComplete/AutoCompleteItem.tsx +++ b/datahub-web-react/src/app/search/autoComplete/AutoCompleteItem.tsx @@ -39,7 +39,7 @@ export default function AutoCompleteItem({ query, entity }: Props) { return ( } placement="top" diff --git a/datahub-web-react/src/app/search/autoComplete/AutoCompleteTooltipContent.tsx b/datahub-web-react/src/app/search/autoComplete/AutoCompleteTooltipContent.tsx index 7a0e104cf1ad6..dfe32c7805a9b 100644 --- a/datahub-web-react/src/app/search/autoComplete/AutoCompleteTooltipContent.tsx +++ b/datahub-web-react/src/app/search/autoComplete/AutoCompleteTooltipContent.tsx @@ -53,7 +53,7 @@ export default function AutoCompleteTooltipContent({ entity }: Props) { } queryCountLast30Days={(entity as Dataset).statsSummary?.queryCountLast30Days} uniqueUserCountLast30Days={(entity as Dataset).statsSummary?.uniqueUserCountLast30Days} - color="" // need to pass in empty color so that tooltip decides the color here + mode="tooltip-content" /> )} diff --git a/datahub-web-react/src/utils/formatter/index.ts b/datahub-web-react/src/utils/formatter/index.ts index 270293960f87d..85919611e2754 100644 --- a/datahub-web-react/src/utils/formatter/index.ts +++ b/datahub-web-react/src/utils/formatter/index.ts @@ -1,20 +1,31 @@ -const intlFormat = (num) => { - return new Intl.NumberFormat().format(Math.round(num * 10) / 10); -}; +type NumMapType = Record<'billion' | 'million' | 'thousand', { value: number; symbol: string }>; -export const countFormatter: (num: number) => string = (num: number) => { - if (num >= 1000000000) { - return `${intlFormat(num / 1000000000)}B`; - } - if (num >= 1000000) { - return `${intlFormat(num / 1000000)}M`; - } +const NumMap: NumMapType = { + billion: { + value: 1000000000, + symbol: 'B', + }, + million: { + value: 1000000, + symbol: 'M', + }, + thousand: { + value: 1000, + symbol: 'K', + }, +} as const; - if (num >= 1000) return `${intlFormat(num / 1000)}K`; +const isBillions = (num: number) => num >= NumMap.billion.value; +const isMillions = (num: number) => num >= NumMap.million.value; +const isThousands = (num: number) => num >= NumMap.thousand.value; - return intlFormat(num); -}; +const intlFormat = (num: number) => new Intl.NumberFormat().format(Math.round(num * 10) / 10); -export const countSeparator = (num) => { - return num.toLocaleString(); +export const needsFormatting = (num: number) => isThousands(num); + +export const countFormatter = (num: number) => { + if (isBillions(num)) return `${intlFormat(num / NumMap.billion.value)}${NumMap.billion.symbol}`; + if (isMillions(num)) return `${intlFormat(num / NumMap.million.value)}${NumMap.million.symbol}`; + if (isThousands(num)) return `${intlFormat(num / NumMap.thousand.value)}${NumMap.thousand.symbol}`; + return intlFormat(num); }; From 9254a1b614f158cfa3a1f0f848c1ed3fcc975b71 Mon Sep 17 00:00:00 2001 From: Indy Prentice Date: Wed, 14 Jun 2023 16:05:54 -0500 Subject: [PATCH 013/222] docs(es): Update aws deploy docs to correct ElasticSearch version (#8240) Co-authored-by: Indy Prentice --- docs/deploy/aws.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deploy/aws.md b/docs/deploy/aws.md index 3fddeb4fd90c1..a8da85c16362d 100644 --- a/docs/deploy/aws.md +++ b/docs/deploy/aws.md @@ -230,7 +230,7 @@ Run `helm upgrade --install datahub datahub/datahub --values values.yaml` to app ### Elasticsearch Service -Provision an elasticsearch domain running elasticsearch version 7.9 or above that shares the VPC with the kubernetes +Provision an elasticsearch domain running elasticsearch version 7.10 or above that shares the VPC with the kubernetes cluster or has VPC peering set up between the VPC of the kubernetes cluster. Once the domain is provisioned, you should be able to see the following page. Take a note of the endpoint marked by the red box. From 2d7692a245f773f7a40bdd2e1503a27326d03afc Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 14 Jun 2023 14:56:27 -0700 Subject: [PATCH 014/222] feat(sdk): support patches as MCPs in file source (#8220) Co-authored-by: Shirshanka Das --- metadata-ingestion/src/datahub/emitter/mcp.py | 5 ++ .../unit/patch/complex_dataset_patch.json | 42 ++++++++++++ .../tests/unit/patch/test_patch_builder.py | 66 +++++++++++++++++++ .../tests/unit/serde/test_serde.py | 7 +- .../tests/unit/serde/test_serde_patch.json | 58 ++++++++++++++++ 5 files changed, 175 insertions(+), 3 deletions(-) create mode 100644 metadata-ingestion/tests/unit/patch/complex_dataset_patch.json create mode 100644 metadata-ingestion/tests/unit/patch/test_patch_builder.py create mode 100644 metadata-ingestion/tests/unit/serde/test_serde_patch.json diff --git a/metadata-ingestion/src/datahub/emitter/mcp.py b/metadata-ingestion/src/datahub/emitter/mcp.py index 027dc768a4779..20e0c659ae283 100644 --- a/metadata-ingestion/src/datahub/emitter/mcp.py +++ b/metadata-ingestion/src/datahub/emitter/mcp.py @@ -197,6 +197,11 @@ def try_from_mcpc( Raises: Exception if the generic aspect is invalid, e.g. contains invalid json. """ + + if mcpc.changeType != ChangeTypeClass.UPSERT: + # We can only generate MCPWs for upserts. + return None + converted, aspect = _try_from_generic_aspect(mcpc.aspectName, mcpc.aspect) if converted: return cls( diff --git a/metadata-ingestion/tests/unit/patch/complex_dataset_patch.json b/metadata-ingestion/tests/unit/patch/complex_dataset_patch.json new file mode 100644 index 0000000000000..08e100140c38b --- /dev/null +++ b/metadata-ingestion/tests/unit/patch/complex_dataset_patch.json @@ -0,0 +1,42 @@ +[ + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "changeType": "PATCH", + "aspectName": "datasetProperties", + "aspect": { + "value": "[{\"op\": \"replace\", \"path\": \"/description\", \"value\": \"test description\"}, {\"op\": \"add\", \"path\": \"/customProperties/test_key_1\", \"value\": \"test_value_1\"}, {\"op\": \"add\", \"path\": \"/customProperties/test_key_2\", \"value\": \"test_value_2\"}]", + "contentType": "application/json-patch+json" + } + }, + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "changeType": "PATCH", + "aspectName": "globalTags", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/tags/urn:li:tag:test_tag\", \"value\": {\"tag\": \"urn:li:tag:test_tag\"}}]", + "contentType": "application/json-patch+json" + } + }, + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cfct_users_created_upstream%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created_upstream,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" + } + }, + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "changeType": "PATCH", + "aspectName": "editableSchemaMetadata", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/editableSchemaFieldInfo/field1/globalTags/tags/urn:li:tag:tag1\", \"value\": {\"tag\": \"urn:li:tag:tag1\"}}]", + "contentType": "application/json-patch+json" + } + } +] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/patch/test_patch_builder.py b/metadata-ingestion/tests/unit/patch/test_patch_builder.py new file mode 100644 index 0000000000000..0701b3d696895 --- /dev/null +++ b/metadata-ingestion/tests/unit/patch/test_patch_builder.py @@ -0,0 +1,66 @@ +import json +import pathlib + +import pytest + +from datahub.emitter.mce_builder import make_dataset_urn, make_tag_urn +from datahub.ingestion.sink.file import write_metadata_file +from datahub.metadata.schema_classes import ( + DatasetLineageTypeClass, + GenericAspectClass, + MetadataChangeProposalClass, + TagAssociationClass, + UpstreamClass, +) +from datahub.specific.dataset import DatasetPatchBuilder + + +def test_basic_dataset_patch_builder(): + patcher = DatasetPatchBuilder( + make_dataset_urn(platform="hive", name="fct_users_created", env="PROD") + ).add_tag(TagAssociationClass(tag=make_tag_urn("test_tag"))) + + assert patcher.build() == [ + MetadataChangeProposalClass( + entityType="dataset", + entityUrn="urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + changeType="PATCH", + aspectName="globalTags", + aspect=GenericAspectClass( + value=b'[{"op": "add", "path": "/tags/urn:li:tag:test_tag", "value": {"tag": "urn:li:tag:test_tag"}}]', + contentType="application/json-patch+json", + ), + ), + ] + + +def test_complex_dataset_patch( + pytestconfig: pytest.Config, tmp_path: pathlib.Path +) -> None: + patcher = ( + DatasetPatchBuilder( + make_dataset_urn(platform="hive", name="fct_users_created", env="PROD") + ) + .set_description("test description") + .add_custom_property("test_key_1", "test_value_1") + .add_custom_property("test_key_2", "test_value_2") + .add_tag(TagAssociationClass(tag=make_tag_urn("test_tag"))) + .add_upstream_lineage( + upstream=UpstreamClass( + dataset=make_dataset_urn( + platform="hive", name="fct_users_created_upstream", env="PROD" + ), + type=DatasetLineageTypeClass.TRANSFORMED, + ) + ) + ) + patcher.for_field("field1").add_tag(TagAssociationClass(tag=make_tag_urn("tag1"))) + + out_path = tmp_path / "patch.json" + write_metadata_file(out_path, patcher.build()) + + assert json.loads(out_path.read_text()) == json.loads( + ( + pytestconfig.rootpath / "tests/unit/patch/complex_dataset_patch.json" + ).read_text() + ) diff --git a/metadata-ingestion/tests/unit/serde/test_serde.py b/metadata-ingestion/tests/unit/serde/test_serde.py index 4be6df15f9450..d116f1f5473fa 100644 --- a/metadata-ingestion/tests/unit/serde/test_serde.py +++ b/metadata-ingestion/tests/unit/serde/test_serde.py @@ -36,15 +36,16 @@ "tests/unit/serde/test_serde_usage.json", # Profiles with the MetadataChangeProposal format. "tests/unit/serde/test_serde_profile.json", + # Test one that uses patch. + "tests/unit/serde/test_serde_patch.json", ], ) def test_serde_to_json( pytestconfig: PytestConfig, tmp_path: pathlib.Path, json_filename: str ) -> None: golden_file = pytestconfig.rootpath / json_filename + output_file = tmp_path / "output.json" - output_filename = "output.json" - output_file = tmp_path / output_filename pipeline = Pipeline.create( { "source": {"type": "file", "config": {"filename": str(golden_file)}}, @@ -57,7 +58,7 @@ def test_serde_to_json( mce_helpers.check_golden_file( pytestconfig, - output_path=f"{tmp_path}/{output_filename}", + output_path=f"{output_file}", golden_path=golden_file, ) diff --git a/metadata-ingestion/tests/unit/serde/test_serde_patch.json b/metadata-ingestion/tests/unit/serde/test_serde_patch.json new file mode 100644 index 0000000000000..33684ea68ea06 --- /dev/null +++ b/metadata-ingestion/tests/unit/serde/test_serde_patch.json @@ -0,0 +1,58 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "changeType": "PATCH", + "aspectName": "datasetProperties", + "aspect": { + "value": "[{\"op\": \"replace\", \"path\": \"/description\", \"value\": \"test description\"}, {\"op\": \"add\", \"path\": \"/customProperties/test_key_1\", \"value\": \"test_value_1\"}, {\"op\": \"add\", \"path\": \"/customProperties/test_key_2\", \"value\": \"test_value_2\"}]", + "contentType": "application/json-patch+json" + }, + "systemMetadata": { + "lastObserved": 1626980046000, + "runId": "serde_test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "changeType": "PATCH", + "aspectName": "globalTags", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/tags/urn:li:tag:test_tag\", \"value\": {\"tag\": \"urn:li:tag:test_tag\"}}]", + "contentType": "application/json-patch+json" + }, + "systemMetadata": { + "lastObserved": 1626980046000, + "runId": "serde_test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cfct_users_created_upstream%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created_upstream,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" + }, + "systemMetadata": { + "lastObserved": 1626980046000, + "runId": "serde_test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "changeType": "PATCH", + "aspectName": "editableSchemaMetadata", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/editableSchemaFieldInfo/field1/globalTags/tags/urn:li:tag:tag1\", \"value\": {\"tag\": \"urn:li:tag:tag1\"}}]", + "contentType": "application/json-patch+json" + }, + "systemMetadata": { + "lastObserved": 1626980046000, + "runId": "serde_test" + } +} +] \ No newline at end of file From 9588440549f3d99965085e97b214a7dabc181ed2 Mon Sep 17 00:00:00 2001 From: RyanHolstien Date: Wed, 14 Jun 2023 17:21:09 -0500 Subject: [PATCH 015/222] fix(apiAuth): add resources where applicable and update docs (#8234) --- docs/authorization/access-policies-guide.md | 65 +++++++++-------- docs/authorization/policies.md | 70 ++++++++++--------- .../authorization/ResolvedResourceSpec.java | 2 + .../datahub/authorization/PolicyEngine.java | 4 ++ .../resources/entity/AspectResource.java | 9 ++- .../resources/entity/EntityResource.java | 8 ++- .../metadata/resources/usage/UsageStats.java | 5 +- 7 files changed, 96 insertions(+), 67 deletions(-) diff --git a/docs/authorization/access-policies-guide.md b/docs/authorization/access-policies-guide.md index 2a4fa7460c91e..5820e513a83e3 100644 --- a/docs/authorization/access-policies-guide.md +++ b/docs/authorization/access-policies-guide.md @@ -193,42 +193,42 @@ scope. The common Metadata Privileges, which span across entity types, include: -| Common Privileges | Description | -|----------------------------------|----------------------------------------------------------------------------------------------------------------------------------| -| View Entity Page | Allow actor to access the entity page for the resource in the UI. If not granted, it will redirect them to an unauthorized page. | -| Edit Tags | Allow actor to add and remove tags to an asset. | -| Edit Glossary Terms | Allow actor to add and remove glossary terms to an asset. | -| Edit Owners | Allow actor to add and remove owners of an entity. | -| Edit Description | Allow actor to edit the description (documentation) of an entity. | -| Edit Links | Allow actor to edit links associated with an entity. | -| Edit Status | Allow actor to edit the status of an entity (soft deleted or not). | -| Edit Domain | Allow actor to edit the Domain of an entity. | -| Edit Deprecation | Allow actor to edit the Deprecation status of an entity. | -| Edit Assertions | Allow actor to add and remove assertions from an entity. | -| Edit All | Allow actor to edit any information about an entity. Super user privileges. | -| Get Timeline API[^1] | Allow actor to get the timeline of an entity via API. | -| Get Entity API[^1] | Allow actor to get an entity via API. | -| Get Timeseries Aspect API[^1] | Allow actor to get a timeseries aspect via API. | -| Get Aspect/Entity Count APIs[^1] | Allow actor to get aspect and entity counts via API. | -| Search API | Allow actor to search for entities via API. | -| Produce Platform Event API | Allow actor to ingest a platform event via API. | +| Common Privileges | Description | +|----------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------| +| View Entity Page | Allow actor to access the entity page for the resource in the UI. If not granted, it will redirect them to an unauthorized page. | +| Edit Tags | Allow actor to add and remove tags to an asset. | +| Edit Glossary Terms | Allow actor to add and remove glossary terms to an asset. | +| Edit Owners | Allow actor to add and remove owners of an entity. | +| Edit Description | Allow actor to edit the description (documentation) of an entity. | +| Edit Links | Allow actor to edit links associated with an entity. | +| Edit Status | Allow actor to edit the status of an entity (soft deleted or not). | +| Edit Domain | Allow actor to edit the Domain of an entity. | +| Edit Deprecation | Allow actor to edit the Deprecation status of an entity. | +| Edit Assertions | Allow actor to add and remove assertions from an entity. | +| Edit All | Allow actor to edit any information about an entity. Super user privileges. Controls the ability to ingest using API when REST API Authorization is enabled. | +| Get Timeline API[^1] | Allow actor to get the timeline of an entity via API. | +| Get Entity API[^1] | Allow actor to get an entity via API. | +| Get Timeseries Aspect API[^1] | Allow actor to get a timeseries aspect via API. | +| Get Aspect/Entity Count APIs[^1] | Allow actor to get aspect and entity counts via API. | +| Search API | Allow actor to search for entities via API. | +| Produce Platform Event API | Allow actor to ingest a platform event via API. | [^1]: Only active if REST_API_AUTHORIZATION_ENABLED is true **Specific Metadata Privileges** include -| Entity | Privilege | Description | -|--------------|------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Dataset | Edit Dataset Column Tags | Allow actor to edit the column (field) tags associated with a dataset schema. | -| Dataset | Edit Dataset Column Glossary Terms | Allow actor to edit the column (field) glossary terms associated with a dataset schema. | -| Dataset | Edit Dataset Column Descriptions | Allow actor to edit the column (field) descriptions associated with a dataset schema. | -| Dataset | Edit Dataset Queries | Allow actor to edit the Highlighted Queries on the Queries tab of the dataset. | -| Dataset | View Dataset Usage | Allow actor to access usage metadata about a dataset both in the UI and in the GraphQL API. This includes example queries, number of queries, etc. | -| Dataset | View Dataset Profile | Allow actor to access a dataset's profile both in the UI and in the GraphQL API. This includes snapshot statistics like #rows, #columns, null percentage per field, etc. | -| Tag | Edit Tag Color | Allow actor to change the color of a Tag. | -| Group | Edit Group Members | Allow actor to add and remove members to a group. | -| User | Edit User Profile | Allow actor to change the user's profile including display name, bio, title, profile image, etc. | -| User + Group | Edit Contact Information | Allow actor to change the contact information such as email & chat handles. | +| Entity | Privilege | Description | +|--------------|------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Dataset | Edit Dataset Column Tags | Allow actor to edit the column (field) tags associated with a dataset schema. | +| Dataset | Edit Dataset Column Glossary Terms | Allow actor to edit the column (field) glossary terms associated with a dataset schema. | +| Dataset | Edit Dataset Column Descriptions | Allow actor to edit the column (field) descriptions associated with a dataset schema. | +| Dataset | Edit Dataset Queries | Allow actor to edit the Highlighted Queries on the Queries tab of the dataset. | +| Dataset | View Dataset Usage | Allow actor to access usage metadata about a dataset both in the UI and in the GraphQL API. This includes example queries, number of queries, etc. Also applies to REST APIs when REST API Authorization is enabled. | +| Dataset | View Dataset Profile | Allow actor to access a dataset's profile both in the UI and in the GraphQL API. This includes snapshot statistics like #rows, #columns, null percentage per field, etc. | +| Tag | Edit Tag Color | Allow actor to change the color of a Tag. | +| Group | Edit Group Members | Allow actor to add and remove members to a group. | +| User | Edit User Profile | Allow actor to change the user's profile including display name, bio, title, profile image, etc. | +| User + Group | Edit Contact Information | Allow actor to change the contact information such as email & chat handles. | > **Still have questions about Privileges?** Let us know in [Slack](https://slack.datahubproject.io)! @@ -304,6 +304,9 @@ when they are bootstrapping with DataHub. This way, those setting up DataHub can Note that these Privileges *can* and likely *should* be changed inside the **Policies** page before onboarding your company's users. +### REST API Authorization + +Policies only affect REST APIs when the environment variable `REST_API_AUTHORIZATION` is set to `true` for GMS. Some policies only apply when this setting is enabled, marked above, and other Metadata and Platform policies apply to the APIs where relevant, also specified in the table above. ## Additional Resources diff --git a/docs/authorization/policies.md b/docs/authorization/policies.md index 26d99af7c4b2d..27d8b15e5a73a 100644 --- a/docs/authorization/policies.md +++ b/docs/authorization/policies.md @@ -94,44 +94,43 @@ We currently support the following: **Common metadata privileges** to view & modify any entity within DataHub. -| Common Privileges | Description | -|----------------------------------|----------------------------------------------------------------------------------------------------------------------------------| -| View Entity Page | Allow actor to access the entity page for the resource in the UI. If not granted, it will redirect them to an unauthorized page. | -| Edit Tags | Allow actor to add and remove tags to an asset. | -| Edit Glossary Terms | Allow actor to add and remove glossary terms to an asset. | -| Edit Owners | Allow actor to add and remove owners of an entity. | -| Edit Description | Allow actor to edit the description (documentation) of an entity. | -| Edit Links | Allow actor to edit links associated with an entity. | -| Edit Status | Allow actor to edit the status of an entity (soft deleted or not). | -| Edit Domain | Allow actor to edit the Domain of an entity. | -| Edit Deprecation | Allow actor to edit the Deprecation status of an entity. | -| Edit Assertions | Allow actor to add and remove assertions from an entity. | -| Edit All | Allow actor to edit any information about an entity. Super user privileges. | -| Edit All | Allow actor to edit any information about an entity. Super user privileges. | -| Get Timeline API[^1] | Allow actor to get the timeline of an entity via API. | -| Get Entity API[^1] | Allow actor to get an entity via API. | -| Get Timeseries Aspect API[^1] | Allow actor to get a timeseries aspect via API. | -| Get Aspect/Entity Count APIs[^1] | Allow actor to get aspect and entity counts via API. | -| Search API[^1] | Allow actor to search for entities via API. | -| Produce Platform Event API[^1] | Allow actor to ingest a platform event via API. | +| Common Privileges | Description | +|----------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------| +| View Entity Page | Allow actor to access the entity page for the resource in the UI. If not granted, it will redirect them to an unauthorized page. | +| Edit Tags | Allow actor to add and remove tags to an asset. | +| Edit Glossary Terms | Allow actor to add and remove glossary terms to an asset. | +| Edit Owners | Allow actor to add and remove owners of an entity. | +| Edit Description | Allow actor to edit the description (documentation) of an entity. | +| Edit Links | Allow actor to edit links associated with an entity. | +| Edit Status | Allow actor to edit the status of an entity (soft deleted or not). | +| Edit Domain | Allow actor to edit the Domain of an entity. | +| Edit Deprecation | Allow actor to edit the Deprecation status of an entity. | +| Edit Assertions | Allow actor to add and remove assertions from an entity. | +| Edit All | Allow actor to edit any information about an entity. Super user privileges. Controls the ability to ingest using API when REST API Authorization is enabled. | | +| Get Timeline API[^1] | Allow actor to get the timeline of an entity via API. | +| Get Entity API[^1] | Allow actor to get an entity via API. | +| Get Timeseries Aspect API[^1] | Allow actor to get a timeseries aspect via API. | +| Get Aspect/Entity Count APIs[^1] | Allow actor to get aspect and entity counts via API. | +| Search API[^1] | Allow actor to search for entities via API. | +| Produce Platform Event API[^1] | Allow actor to ingest a platform event via API. | [^1]: Only active if REST_API_AUTHORIZATION_ENABLED is true **Specific entity-level privileges** that are not generalizable. -| Entity | Privilege | Description | -|--------------|------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Dataset | Edit Dataset Column Tags | Allow actor to edit the column (field) tags associated with a dataset schema. | -| Dataset | Edit Dataset Column Glossary Terms | Allow actor to edit the column (field) glossary terms associated with a dataset schema. | -| Dataset | Edit Dataset Column Descriptions | Allow actor to edit the column (field) descriptions associated with a dataset schema. | -| Dataset | View Dataset Usage | Allow actor to access usage metadata about a dataset both in the UI and in the GraphQL API. This includes example queries, number of queries, etc. | -| Dataset | View Dataset Profile | Allow actor to access a dataset's profile both in the UI and in the GraphQL API. This includes snapshot statistics like #rows, #columns, null percentage per field, etc. | -| Tag | Edit Tag Color | Allow actor to change the color of a Tag. | -| Group | Edit Group Members | Allow actor to add and remove members to a group. | -| User | Edit User Profile | Allow actor to change the user's profile including display name, bio, title, profile image, etc. | -| User + Group | Edit Contact Information | Allow actor to change the contact information such as email & chat handles. | -| GlossaryNode | Manage Direct Glossary Children | Allow the actor to create, edit, and delete the direct children of the selected entities. | -| GlossaryNode | Manage All Glossary Children | Allow the actor to create, edit, and delete everything underneath the selected entities. | +| Entity | Privilege | Description | +|--------------|------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Dataset | Edit Dataset Column Tags | Allow actor to edit the column (field) tags associated with a dataset schema. | +| Dataset | Edit Dataset Column Glossary Terms | Allow actor to edit the column (field) glossary terms associated with a dataset schema. | +| Dataset | Edit Dataset Column Descriptions | Allow actor to edit the column (field) descriptions associated with a dataset schema. | +| Dataset | View Dataset Usage | Allow actor to access usage metadata about a dataset both in the UI and in the GraphQL API. This includes example queries, number of queries, etc. Also applies to REST APIs when REST API Authorization is enabled. | +| Dataset | View Dataset Profile | Allow actor to access a dataset's profile both in the UI and in the GraphQL API. This includes snapshot statistics like #rows, #columns, null percentage per field, etc. | +| Tag | Edit Tag Color | Allow actor to change the color of a Tag. | +| Group | Edit Group Members | Allow actor to add and remove members to a group. | +| User | Edit User Profile | Allow actor to change the user's profile including display name, bio, title, profile image, etc. | +| User + Group | Edit Contact Information | Allow actor to change the contact information such as email & chat handles. | +| GlossaryNode | Manage Direct Glossary Children | Allow the actor to create, edit, and delete the direct children of the selected entities. | +| GlossaryNode | Manage All Glossary Children | Allow the actor to create, edit, and delete everything underneath the selected entities. | @@ -216,6 +215,11 @@ to `false`. For example in your `docker/datahub-gms/docker.env`, you'd place AUTH_POLICIES_ENABLED=false ``` +### REST API Authorization + +Policies only affect REST APIs when the environment variable `REST_API_AUTHORIZATION` is set to `true` for GMS. Some policies only apply when this setting is enabled, marked above, and other Metadata and Platform policies apply to the APIs where relevant, also specified in the table above. + + ## Coming Soon The DataHub team is hard at work trying to improve the Policies feature. We are planning on building out the following: diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java index 21369c19275c7..0dae1bd386ccd 100644 --- a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java +++ b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java @@ -6,12 +6,14 @@ import javax.annotation.Nullable; import lombok.Getter; import lombok.RequiredArgsConstructor; +import lombok.ToString; /** * Wrapper around authorization request with field resolvers for lazily fetching the field values for each field type */ @RequiredArgsConstructor +@ToString public class ResolvedResourceSpec { @Getter private final ResourceSpec spec; diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java index 45e3e4b8b188b..44b0c0d56f6fe 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java @@ -64,14 +64,17 @@ public PolicyEvaluationResult evaluatePolicy( final Optional resource) { final PolicyEvaluationContext context = new PolicyEvaluationContext(); + log.debug("Evaluating policy {}", policy.getDisplayName()); // If the privilege is not in scope, deny the request. if (!isPrivilegeMatch(privilege, policy.getPrivileges(), context)) { + log.debug("Policy denied based on irrelevant privileges {} for {}", policy.getPrivileges(), privilege); return PolicyEvaluationResult.DENIED; } // If policy is not applicable, deny the request if (!isPolicyApplicable(policy, actor, resource, context)) { + log.debug("Policy does not applicable for actor {} and resource {}", actor, resource); return PolicyEvaluationResult.DENIED; } @@ -193,6 +196,7 @@ private boolean isResourceMatch( } if (!requestResource.isPresent()) { // Resource filter present in policy, but no resource spec provided. + log.debug("Resource filter present in policy, but no resource spec provided."); return false; } final PolicyMatchFilter filter = getFilter(policyResourceFilter); diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java index 8df7629d94716..3b92c78c8d6bc 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java @@ -9,6 +9,7 @@ import com.linkedin.aspect.GetTimeseriesAspectValuesResponse; import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.metadata.aspect.EnvelopedAspectArray; import com.linkedin.metadata.aspect.VersionedAspect; import com.linkedin.metadata.authorization.PoliciesConfig; @@ -46,6 +47,7 @@ import javax.inject.Inject; import javax.inject.Named; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import static com.linkedin.metadata.Constants.*; import static com.linkedin.metadata.resources.restli.RestliConstants.*; @@ -234,9 +236,14 @@ public Task restoreIndices(@ActionParam(PARAM_ASPECT) @Optional @Nonnull ) { return RestliUtil.toTask(() -> { Authentication authentication = AuthenticationContext.getAuthentication(); + ResourceSpec resourceSpec = null; + if (StringUtils.isNotBlank(urn)) { + Urn resource = UrnUtils.getUrn(urn); + resourceSpec = new ResourceSpec(resource.getEntityType(), resource.toString()); + } if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV)) && !isAuthorized(authentication, _authorizer, ImmutableList.of(PoliciesConfig.RESTORE_INDICES_PRIVILEGE), - (ResourceSpec) null)) { + resourceSpec)) { throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED, "User is unauthorized to restore indices."); } RestoreIndicesArgs args = new RestoreIndicesArgs() diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java index 89ef7fb4560b3..395d802651e70 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java @@ -77,6 +77,7 @@ import javax.inject.Inject; import javax.inject.Named; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.apache.maven.artifact.versioning.ComparableVersion; import static com.linkedin.metadata.Constants.*; @@ -755,8 +756,13 @@ public Task applyRetention(@ActionParam(PARAM_START) @Optional @Nullable @ActionParam(PARAM_URN) @Optional @Nullable String urn ) { Authentication auth = AuthenticationContext.getAuthentication(); + ResourceSpec resourceSpec = null; + if (StringUtils.isNotBlank(urn)) { + Urn resource = UrnUtils.getUrn(urn); + resourceSpec = new ResourceSpec(resource.getEntityType(), resource.toString()); + } if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV)) - && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.APPLY_RETENTION_PRIVILEGE), (ResourceSpec) null)) { + && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.APPLY_RETENTION_PRIVILEGE), resourceSpec)) { throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED, "User is unauthorized to apply retention."); } diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java index 84cdbc5d4f415..ddfdec0315f6b 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java @@ -12,6 +12,7 @@ import com.google.common.collect.ImmutableList; import com.linkedin.common.WindowDuration; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.StringArray; import com.linkedin.dataset.DatasetFieldUsageCounts; import com.linkedin.dataset.DatasetFieldUsageCountsArray; @@ -319,8 +320,10 @@ public Task query(@ActionParam(PARAM_RESOURCE) @Nonnull String log.info("Attempting to query usage stats"); return RestliUtil.toTask(() -> { Authentication auth = AuthenticationContext.getAuthentication(); + Urn resourceUrn = UrnUtils.getUrn(resource); if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV)) - && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.VIEW_DATASET_USAGE_PRIVILEGE), (ResourceSpec) null)) { + && !isAuthorized(auth, _authorizer, ImmutableList.of(PoliciesConfig.VIEW_DATASET_USAGE_PRIVILEGE), + new ResourceSpec(resourceUrn.getEntityType(), resourceUrn.toString()))) { throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED, "User is unauthorized to query usage."); } From f78f7f960943b890e2cb6f48cc46370e90d0380e Mon Sep 17 00:00:00 2001 From: RyanHolstien Date: Thu, 15 Jun 2023 00:21:16 -0500 Subject: [PATCH 016/222] feat(patch): support datajob input output (#8190) Co-authored-by: Shirshanka Das Co-authored-by: david-leifker <114954101+david-leifker@users.noreply.github.com> --- docs-website/sidebars.js | 1 + docs/advanced/patch.md | 64 ++ .../registry/SnapshotEntityRegistry.java | 2 + .../template/AspectTemplateEngine.java | 3 +- .../datajob/DataJobInputOutputTemplate.java | 95 +++ .../src/datahub/specific/datajob.py | 670 ++++++++++++++++++ .../DataJobInputOutputPatchBuilder.java | 214 ++++++ .../java/datahub/client/patch/PatchTest.java | 83 +++ .../examples/DataJobLineageAdd.java | 59 ++ smoke-test/tests/patch/common_patch_tests.py | 277 ++++++++ .../tests/patch/test_datajob_patches.py | 135 ++++ .../tests/patch/test_dataset_patches.py | 237 +------ 12 files changed, 1619 insertions(+), 221 deletions(-) create mode 100644 docs/advanced/patch.md create mode 100644 entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/datajob/DataJobInputOutputTemplate.java create mode 100644 metadata-ingestion/src/datahub/specific/datajob.py create mode 100644 metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/datajob/DataJobInputOutputPatchBuilder.java create mode 100644 metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DataJobLineageAdd.java create mode 100644 smoke-test/tests/patch/common_patch_tests.py create mode 100644 smoke-test/tests/patch/test_datajob_patches.py diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index 43537c57433b4..269f5dfe9dedf 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -316,6 +316,7 @@ module.exports = { "metadata-integration/java/as-a-library", { "API and SDK Guides": [ + "docs/advanced/patch", "docs/api/tutorials/datasets", "docs/api/tutorials/lineage", "docs/api/tutorials/tags", diff --git a/docs/advanced/patch.md b/docs/advanced/patch.md new file mode 100644 index 0000000000000..21c28941c265a --- /dev/null +++ b/docs/advanced/patch.md @@ -0,0 +1,64 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# But First, Semantics: Upsert versus Patch + +## Why Would You Use Patch + +By default, most of the SDK tutorials and API-s involve applying full upserts at the aspect level. This means that typically, when you want to change one field within an aspect without modifying others, you need to do a read-modify-write to not overwrite existing fields. +To support these scenarios, DataHub supports PATCH based operations so that targeted changes to single fields or values within arrays of fields are possible without impacting other existing metadata. + +:::note + +Currently, PATCH support is only available for a selected set of aspects, so before pinning your hopes on using PATCH as a way to make modifications to aspect values, confirm whether your aspect supports PATCH semantics. The complete list of Aspects that are supported are maintained [here](https://github.com/datahub-project/datahub/blob/9588440549f3d99965085e97b214a7dabc181ed2/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/AspectTemplateEngine.java#L24). In the near future, we do have plans to automatically support PATCH semantics for aspects by default. + +::: + +## How To Use Patch + +Examples for using Patch are sprinkled throughout the API guides. +Here's how to find the appropriate classes for the language for your choice. + + + + + +The Java Patch builders are aspect-oriented and located in the [datahub-client](https://github.com/datahub-project/datahub/tree/master/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch) module under the `datahub.client.patch` namespace. + +Here are a few illustrative examples using the Java Patch builders: + + +### Add Custom Properties + +```java +{{ inline /metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetCustomPropertiesAdd.java show_path_as_comment }} +``` + +### Add and Remove Custom Properties + +```java +{{ inline /metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetCustomPropertiesAddRemove.java show_path_as_comment }} +``` + +### Add Data Job Lineage + +```java +{{ inline /metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DataJobLineageAdd.java show_path_as_comment }} +``` + + + + +The Python Patch builders are entity-oriented and located in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/9588440549f3d99965085e97b214a7dabc181ed2/metadata-ingestion/src/datahub/specific) module and located in the `datahub.specific` module. + +Here are a few illustrative examples using the Python Patch builders: + +### Add Properties to Dataset + +```python +{{ inline /metadata-ingestion/examples/library/dataset_add_properties.py show_path_as_comment }} +``` + + + + diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/SnapshotEntityRegistry.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/SnapshotEntityRegistry.java index cd64bf200f9ae..32738d65573fd 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/SnapshotEntityRegistry.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/SnapshotEntityRegistry.java @@ -14,6 +14,7 @@ import com.linkedin.metadata.models.registry.template.common.OwnershipTemplate; import com.linkedin.metadata.models.registry.template.dataflow.DataFlowInfoTemplate; import com.linkedin.metadata.models.registry.template.datajob.DataJobInfoTemplate; +import com.linkedin.metadata.models.registry.template.datajob.DataJobInputOutputTemplate; import com.linkedin.metadata.models.registry.template.dataproduct.DataProductPropertiesTemplate; import com.linkedin.metadata.models.registry.template.dataset.DatasetPropertiesTemplate; import com.linkedin.metadata.models.registry.template.dataset.EditableSchemaMetadataTemplate; @@ -74,6 +75,7 @@ private AspectTemplateEngine populateTemplateEngine(Map aspe aspectSpecTemplateMap.put(DATA_FLOW_INFO_ASPECT_NAME, new DataFlowInfoTemplate()); aspectSpecTemplateMap.put(DATA_JOB_INFO_ASPECT_NAME, new DataJobInfoTemplate()); aspectSpecTemplateMap.put(DATA_PRODUCT_PROPERTIES_ASPECT_NAME, new DataProductPropertiesTemplate()); + aspectSpecTemplateMap.put(DATA_JOB_INPUT_OUTPUT_ASPECT_NAME, new DataJobInputOutputTemplate()); return new AspectTemplateEngine(aspectSpecTemplateMap); } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/AspectTemplateEngine.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/AspectTemplateEngine.java index 09aa095c209ae..742dbd70d4503 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/AspectTemplateEngine.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/AspectTemplateEngine.java @@ -30,7 +30,8 @@ public class AspectTemplateEngine { UPSTREAM_LINEAGE_ASPECT_NAME, DATA_FLOW_INFO_ASPECT_NAME, DATA_JOB_INFO_ASPECT_NAME, - DATA_PRODUCT_PROPERTIES_ASPECT_NAME).collect(Collectors.toSet()); + DATA_PRODUCT_PROPERTIES_ASPECT_NAME, + DATA_JOB_INPUT_OUTPUT_ASPECT_NAME).collect(Collectors.toSet()); private final Map> _aspectTemplateMap; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/datajob/DataJobInputOutputTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/datajob/DataJobInputOutputTemplate.java new file mode 100644 index 0000000000000..b4ddb4523c9a5 --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/datajob/DataJobInputOutputTemplate.java @@ -0,0 +1,95 @@ +package com.linkedin.metadata.models.registry.template.datajob; + +import com.fasterxml.jackson.databind.JsonNode; +import com.linkedin.common.DataJobUrnArray; +import com.linkedin.common.DatasetUrnArray; +import com.linkedin.common.EdgeArray; +import com.linkedin.common.UrnArray; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.datajob.DataJobInputOutput; +import com.linkedin.dataset.FineGrainedLineageArray; +import com.linkedin.metadata.models.registry.template.ArrayMergingTemplate; +import java.util.Collections; +import javax.annotation.Nonnull; + + +public class DataJobInputOutputTemplate implements ArrayMergingTemplate { + + private static final String INPUT_DATA_JOB_EDGES_FIELD_NAME = "inputDatajobEdges"; + private static final String INPUT_DATASET_EDGES_FIELD_NAME = "inputDatasetEdges"; + private static final String OUTPUT_DATASET_EDGES_FIELD_NAME = "outputDatasetEdges"; + + private static final String DESTINATION_URN_FIELD_NAME = "destinationUrn"; + + private static final String INPUT_DATASET_FIELDS_FIELD_NAME = "inputDatasetFields"; + private static final String OUTPUT_DATASET_FIELDS_FIELD_NAME = "outputDatasetFields"; + // TODO: Fine Grained Lineages not patchable at this time, they don't have a well established key + + @Override + public DataJobInputOutput getSubtype(RecordTemplate recordTemplate) throws ClassCastException { + if (recordTemplate instanceof DataJobInputOutput) { + return (DataJobInputOutput) recordTemplate; + } + throw new ClassCastException("Unable to cast RecordTemplate to DataJobInputOutput"); + } + + @Override + public Class getTemplateType() { + return DataJobInputOutput.class; + } + + @Nonnull + @Override + public DataJobInputOutput getDefault() { + DataJobInputOutput dataJobInputOutput = new DataJobInputOutput(); + dataJobInputOutput.setInputDatajobEdges(new EdgeArray()); + dataJobInputOutput.setInputDatasetEdges(new EdgeArray()); + dataJobInputOutput.setOutputDatasetEdges(new EdgeArray()); + dataJobInputOutput.setFineGrainedLineages(new FineGrainedLineageArray()); + dataJobInputOutput.setInputDatasetFields(new UrnArray()); + dataJobInputOutput.setOutputDatasetFields(new UrnArray()); + + // Deprecated fields + dataJobInputOutput.setInputDatajobs(new DataJobUrnArray()); + dataJobInputOutput.setInputDatasets(new DatasetUrnArray()); + dataJobInputOutput.setOutputDatasets(new DatasetUrnArray()); + + return dataJobInputOutput; + } + + @Nonnull + @Override + public JsonNode transformFields(JsonNode baseNode) { + JsonNode transformedNode = arrayFieldToMap(baseNode, INPUT_DATA_JOB_EDGES_FIELD_NAME, + Collections.singletonList(DESTINATION_URN_FIELD_NAME)); + + transformedNode = arrayFieldToMap(transformedNode, INPUT_DATASET_EDGES_FIELD_NAME, + Collections.singletonList(DESTINATION_URN_FIELD_NAME)); + + transformedNode = arrayFieldToMap(transformedNode, OUTPUT_DATASET_EDGES_FIELD_NAME, + Collections.singletonList(DESTINATION_URN_FIELD_NAME)); + + transformedNode = arrayFieldToMap(transformedNode, INPUT_DATASET_FIELDS_FIELD_NAME, Collections.emptyList()); + transformedNode = arrayFieldToMap(transformedNode, OUTPUT_DATASET_FIELDS_FIELD_NAME, Collections.emptyList()); + + return transformedNode; + } + + @Nonnull + @Override + public JsonNode rebaseFields(JsonNode patched) { + JsonNode rebasedNode = transformedMapToArray(patched, INPUT_DATA_JOB_EDGES_FIELD_NAME, + Collections.singletonList(DESTINATION_URN_FIELD_NAME)); + + rebasedNode = transformedMapToArray(rebasedNode, INPUT_DATASET_EDGES_FIELD_NAME, + Collections.singletonList(DESTINATION_URN_FIELD_NAME)); + + rebasedNode = transformedMapToArray(rebasedNode, OUTPUT_DATASET_EDGES_FIELD_NAME, + Collections.singletonList(DESTINATION_URN_FIELD_NAME)); + + rebasedNode = transformedMapToArray(rebasedNode, INPUT_DATASET_FIELDS_FIELD_NAME, Collections.emptyList()); + rebasedNode = transformedMapToArray(rebasedNode, OUTPUT_DATASET_FIELDS_FIELD_NAME, Collections.emptyList()); + + return rebasedNode; + } +} diff --git a/metadata-ingestion/src/datahub/specific/datajob.py b/metadata-ingestion/src/datahub/specific/datajob.py new file mode 100644 index 0000000000000..7ebaee6b918c1 --- /dev/null +++ b/metadata-ingestion/src/datahub/specific/datajob.py @@ -0,0 +1,670 @@ +import time +from typing import Dict, List, Optional, TypeVar, Union +from urllib.parse import quote + +from datahub.emitter.mcp_patch_builder import MetadataPatchProposal +from datahub.metadata.schema_classes import ( + AuditStampClass, + DataJobInfoClass as DataJobInfo, + DataJobInputOutputClass as DataJobInputOutput, + EdgeClass as Edge, + GlobalTagsClass as GlobalTags, + GlossaryTermAssociationClass as Term, + GlossaryTermsClass as GlossaryTerms, + KafkaAuditHeaderClass, + OwnerClass as Owner, + OwnershipTypeClass, + SystemMetadataClass, + TagAssociationClass as Tag, +) +from datahub.specific.custom_properties import CustomPropertiesPatchHelper +from datahub.specific.ownership import OwnershipPatchHelper +from datahub.utilities.urns.tag_urn import TagUrn +from datahub.utilities.urns.urn import Urn + +T = TypeVar("T", bound=MetadataPatchProposal) + + +class DataJobPatchBuilder(MetadataPatchProposal): + def __init__( + self, + urn: str, + system_metadata: Optional[SystemMetadataClass] = None, + audit_header: Optional[KafkaAuditHeaderClass] = None, + ) -> None: + """ + Initializes a DataJobPatchBuilder instance. + + Args: + urn: The URN of the data job. + system_metadata: The system metadata of the data job (optional). + audit_header: The Kafka audit header of the data job (optional). + """ + super().__init__( + urn, "datajob", system_metadata=system_metadata, audit_header=audit_header + ) + self.custom_properties_patch_helper = CustomPropertiesPatchHelper( + self, DataJobInfo.ASPECT_NAME + ) + self.ownership_patch_helper = OwnershipPatchHelper(self) + + def _mint_auditstamp(self, message: Optional[str] = None) -> AuditStampClass: + """ + Creates an AuditStampClass instance with the current timestamp and other default values. + + Args: + message: The message associated with the audit stamp (optional). + + Returns: + An instance of AuditStampClass. + """ + return AuditStampClass( + time=int(time.time() * 1000.0), + actor="urn:li:corpuser:datahub", + message=message, + ) + + def _ensure_urn_type( + self, entity_type: str, edges: List[Edge], context: str + ) -> None: + """ + Ensures that the destination URNs in the given edges have the specified entity type. + + Args: + entity_type: The entity type to check against. + edges: A list of Edge objects. + context: The context or description of the operation. + + Raises: + ValueError: If any of the destination URNs is not of the specified entity type. + """ + for e in edges: + urn = Urn.create_from_string(e.destinationUrn) + if not urn.get_type() == entity_type: + raise ValueError( + f"{context}: {e.destinationUrn} is not of type {entity_type}" + ) + + def add_owner(self, owner: Owner) -> "DataJobPatchBuilder": + """ + Adds an owner to the DataJobPatchBuilder. + + Args: + owner: The Owner object to add. + + Returns: + The DataJobPatchBuilder instance. + """ + self.ownership_patch_helper.add_owner(owner) + return self + + def remove_owner( + self, owner: str, owner_type: Optional[OwnershipTypeClass] = None + ) -> "DataJobPatchBuilder": + """ + Removes an owner from the DataJobPatchBuilder. + + Args: + owner: The owner to remove. + owner_type: The ownership type of the owner (optional). + + Returns: + The DataJobPatchBuilder instance. + + Notes: + `owner_type` is optional. + """ + self.ownership_patch_helper.remove_owner(owner, owner_type) + return self + + def set_owners(self, owners: List[Owner]) -> "DataJobPatchBuilder": + """ + Sets the owners of the DataJobPatchBuilder. + + Args: + owners: A list of Owner objects. + + Returns: + The DataJobPatchBuilder instance. + """ + self.ownership_patch_helper.set_owners(owners) + return self + + def add_input_datajob(self, input: Union[Edge, Urn, str]) -> "DataJobPatchBuilder": + """ + Adds an input data job to the DataJobPatchBuilder. + + Args: + input: The input data job, which can be an Edge object, Urn object, or a string. + + Returns: + The DataJobPatchBuilder instance. + + Raises: + ValueError: If the input is not a DataJob urn. + + Notes: + If `input` is an Edge object, it is used directly. If `input` is a Urn object or string, + it is converted to an Edge object and added with default audit stamps. + """ + if isinstance(input, Edge): + input_urn: str = input.destinationUrn + input_edge: Edge = input + elif isinstance(input, (Urn, str)): + input_urn = str(input) + if not input_urn.startswith("urn:li:dataJob:"): + raise ValueError(f"Input {input} is not a DataJob urn") + + input_edge = Edge( + destinationUrn=input_urn, + created=self._mint_auditstamp(), + lastModified=self._mint_auditstamp(), + ) + + self._ensure_urn_type("dataJob", [input_edge], "add_input_datajob") + self._add_patch( + DataJobInputOutput.ASPECT_NAME, + "add", + path=f"/inputDatajobEdges/{quote(input_urn, safe='')}", + value=input_edge, + ) + return self + + def remove_input_datajob(self, input: Union[str, Urn]) -> "DataJobPatchBuilder": + """ + Removes an input data job from the DataJobPatchBuilder. + + Args: + input: The input data job to remove, specified as a string or Urn object. + + Returns: + The DataJobPatchBuilder instance. + """ + self._add_patch( + DataJobInputOutput.ASPECT_NAME, + "remove", + path=f"/inputDatajobEdges/{input}", + value={}, + ) + return self + + def set_input_datajobs(self, inputs: List[Edge]) -> "DataJobPatchBuilder": + """ + Sets the input data jobs for the DataJobPatchBuilder. + + Args: + inputs: A list of Edge objects representing the input data jobs. + + Returns: + The DataJobPatchBuilder instance. + + Raises: + ValueError: If any of the input edges are not of type 'dataJob'. + + Notes: + This method replaces all existing input data jobs with the given inputs. + """ + self._ensure_urn_type("dataJob", inputs, "input datajobs") + self._add_patch( + DataJobInputOutput.ASPECT_NAME, + "replace", + path="/inputDatajobEdges", + value=inputs, + ) + return self + + def add_input_dataset(self, input: Union[Edge, Urn, str]) -> "DataJobPatchBuilder": + """ + Adds an input dataset to the DataJobPatchBuilder. + + Args: + input: The input dataset, which can be an Edge object, Urn object, or a string. + + Returns: + The DataJobPatchBuilder instance. + + Raises: + ValueError: If the input is not a Dataset urn. + + Notes: + If `input` is an Edge object, it is used directly. If `input` is a Urn object or string, + it is converted to an Edge object and added with default audit stamps. + """ + if isinstance(input, Edge): + input_urn: str = input.destinationUrn + input_edge: Edge = input + elif isinstance(input, (Urn, str)): + input_urn = str(input) + if not input_urn.startswith("urn:li:dataset:"): + raise ValueError(f"Input {input} is not a Dataset urn") + + input_edge = Edge( + destinationUrn=input_urn, + created=self._mint_auditstamp(), + lastModified=self._mint_auditstamp(), + ) + + self._ensure_urn_type("dataset", [input_edge], "add_input_dataset") + self._add_patch( + DataJobInputOutput.ASPECT_NAME, + "add", + path=f"/inputDatasetEdges/{quote(input_urn, safe='')}", + value=input_edge, + ) + return self + + def remove_input_dataset(self, input: Union[str, Urn]) -> "DataJobPatchBuilder": + """ + Removes an input dataset from the DataJobPatchBuilder. + + Args: + input: The input dataset to remove, specified as a string or Urn object. + + Returns: + The DataJobPatchBuilder instance. + """ + self._add_patch( + DataJobInputOutput.ASPECT_NAME, + "remove", + path=f"/inputDatasetEdges/{input}", + value={}, + ) + return self + + def set_input_datasets(self, inputs: List[Edge]) -> "DataJobPatchBuilder": + """ + Sets the input datasets for the DataJobPatchBuilder. + + Args: + inputs: A list of Edge objects representing the input datasets. + + Returns: + The DataJobPatchBuilder instance. + + Raises: + ValueError: If any of the input edges are not of type 'dataset'. + + Notes: + This method replaces all existing input datasets with the given inputs. + """ + self._ensure_urn_type("dataset", inputs, "set_input_datasets") + self._add_patch( + DataJobInputOutput.ASPECT_NAME, + "replace", + path="/inputDatasetEdges", + value=inputs, + ) + return self + + def add_output_dataset( + self, output: Union[Edge, Urn, str] + ) -> "DataJobPatchBuilder": + """ + Adds an output dataset to the DataJobPatchBuilder. + + Args: + output: The output dataset, which can be an Edge object, Urn object, or a string. + + Returns: + The DataJobPatchBuilder instance. + + Raises: + ValueError: If the output is not a Dataset urn. + + Notes: + If `output` is an Edge object, it is used directly. If `output` is a Urn object or string, + it is converted to an Edge object and added with default audit stamps. + """ + if isinstance(output, Edge): + output_urn: str = output.destinationUrn + output_edge: Edge = output + elif isinstance(output, (Urn, str)): + output_urn = str(output) + if not output_urn.startswith("urn:li:dataset:"): + raise ValueError(f"Input {input} is not a Dataset urn") + + output_edge = Edge( + destinationUrn=output_urn, + created=self._mint_auditstamp(), + lastModified=self._mint_auditstamp(), + ) + + self._ensure_urn_type("dataset", [output_edge], "add_output_dataset") + self._add_patch( + DataJobInputOutput.ASPECT_NAME, + "add", + path=f"/outputDatasetEdges/{quote(output_urn, safe='')}", + value=output_edge, + ) + return self + + def remove_output_dataset(self, output: Union[str, Urn]) -> "DataJobPatchBuilder": + """ + Removes an output dataset from the DataJobPatchBuilder. + + Args: + output: The output dataset to remove, specified as a string or Urn object. + + Returns: + The DataJobPatchBuilder instance. + """ + self._add_patch( + DataJobInputOutput.ASPECT_NAME, + "remove", + path=f"/outputDatasetEdges/{output}", + value={}, + ) + return self + + def set_output_datasets(self, outputs: List[Edge]) -> "DataJobPatchBuilder": + """ + Sets the output datasets for the DataJobPatchBuilder. + + Args: + outputs: A list of Edge objects representing the output datasets. + + Returns: + The DataJobPatchBuilder instance. + + Raises: + ValueError: If any of the output edges are not of type 'dataset'. + + Notes: + This method replaces all existing output datasets with the given outputs. + """ + self._ensure_urn_type("dataset", outputs, "set_output_datasets") + self._add_patch( + DataJobInputOutput.ASPECT_NAME, + "replace", + path="/outputDatasetEdges", + value=outputs, + ) + return self + + def add_input_dataset_field( + self, input: Union[Edge, Urn, str] + ) -> "DataJobPatchBuilder": + """ + Adds an input dataset field to the DataJobPatchBuilder. + + Args: + input: The input dataset field, which can be an Edge object, Urn object, or a string. + + Returns: + The DataJobPatchBuilder instance. + + Raises: + ValueError: If the input is not a Schema Field urn. + + Notes: + If `input` is an Edge object, it is used directly. If `input` is a Urn object or string, + it is converted to an Edge object and added with default audit stamps. + """ + if isinstance(input, Edge): + input_urn: str = input.destinationUrn + input_edge: Edge = input + elif isinstance(input, (Urn, str)): + input_urn = str(input) + if not input_urn.startswith("urn:li:schemaField:"): + raise ValueError(f"Input {input} is not a Schema Field urn") + + input_edge = Edge( + destinationUrn=input_urn, + created=self._mint_auditstamp(), + lastModified=self._mint_auditstamp(), + ) + + self._ensure_urn_type("schemaField", [input_edge], "add_input_dataset_field") + self._add_patch( + DataJobInputOutput.ASPECT_NAME, + "add", + path=f"/inputDatasetFields/{quote(input_urn, safe='')}", + value=input_edge, + ) + return self + + def remove_input_dataset_field( + self, input: Union[str, Urn] + ) -> "DataJobPatchBuilder": + """ + Removes an input dataset field from the DataJobPatchBuilder. + + Args: + input: The input dataset field to remove, specified as a string or Urn object. + + Returns: + The DataJobPatchBuilder instance. + """ + input_urn = str(input) + self._add_patch( + DataJobInputOutput.ASPECT_NAME, + "remove", + path=f"/inputDatasetFields/{quote(input_urn, safe='')}", + value={}, + ) + return self + + def set_input_dataset_fields(self, inputs: List[Edge]) -> "DataJobPatchBuilder": + """ + Sets the input dataset fields for the DataJobPatchBuilder. + + Args: + inputs: A list of Edge objects representing the input dataset fields. + + Returns: + The DataJobPatchBuilder instance. + + Raises: + ValueError: If any of the input edges are not of type 'schemaField'. + + Notes: + This method replaces all existing input dataset fields with the given inputs. + """ + self._ensure_urn_type("schemaField", inputs, "set_input_dataset_fields") + self._add_patch( + DataJobInputOutput.ASPECT_NAME, + "replace", + path="/inputDatasetFields", + value=inputs, + ) + return self + + def add_output_dataset_field( + self, output: Union[Edge, Urn, str] + ) -> "DataJobPatchBuilder": + """ + Adds an output dataset field to the DataJobPatchBuilder. + + Args: + output: The output dataset field, which can be an Edge object, Urn object, or a string. + + Returns: + The DataJobPatchBuilder instance. + + Raises: + ValueError: If the output is not a Schema Field urn. + + Notes: + If `output` is an Edge object, it is used directly. If `output` is a Urn object or string, + it is converted to an Edge object and added with default audit stamps. + """ + if isinstance(output, Edge): + output_urn: str = output.destinationUrn + output_edge: Edge = output + elif isinstance(output, (Urn, str)): + output_urn = str(output) + if not output_urn.startswith("urn:li:schemaField:"): + raise ValueError(f"Input {input} is not a Schema Field urn") + + output_edge = Edge( + destinationUrn=output_urn, + created=self._mint_auditstamp(), + lastModified=self._mint_auditstamp(), + ) + + self._ensure_urn_type("schemaField", [output_edge], "add_output_dataset_field") + self._add_patch( + DataJobInputOutput.ASPECT_NAME, + "add", + path=f"/outputDatasetFields/{quote(output_urn, safe='')}", + value=output_edge, + ) + return self + + def remove_output_dataset_field( + self, output: Union[str, Urn] + ) -> "DataJobPatchBuilder": + """ + Removes an output dataset field from the DataJobPatchBuilder. + + Args: + output: The output dataset field to remove, specified as a string or Urn object. + + Returns: + The DataJobPatchBuilder instance. + """ + output_urn = str(output) + self._add_patch( + DataJobInputOutput.ASPECT_NAME, + "remove", + path=f"/outputDatasetFields/{quote(output_urn, safe='')}", + value={}, + ) + return self + + def set_output_dataset_fields(self, outputs: List[Edge]) -> "DataJobPatchBuilder": + """ + Sets the output dataset fields for the DataJobPatchBuilder. + + Args: + outputs: A list of Edge objects representing the output dataset fields. + + Returns: + The DataJobPatchBuilder instance. + + Raises: + ValueError: If any of the output edges are not of type 'schemaField'. + + Notes: + This method replaces all existing output dataset fields with the given outputs. + """ + self._ensure_urn_type("schemaField", outputs, "set_output_dataset_fields") + self._add_patch( + DataJobInputOutput.ASPECT_NAME, + "replace", + path="/outputDatasetFields", + value=outputs, + ) + return self + + def add_tag(self, tag: Tag) -> "DataJobPatchBuilder": + """ + Adds a tag to the DataJobPatchBuilder. + + Args: + tag: The Tag object representing the tag to be added. + + Returns: + The DataJobPatchBuilder instance. + """ + self._add_patch( + GlobalTags.ASPECT_NAME, "add", path=f"/tags/{tag.tag}", value=tag + ) + return self + + def remove_tag(self, tag: Union[str, Urn]) -> "DataJobPatchBuilder": + """ + Removes a tag from the DataJobPatchBuilder. + + Args: + tag: The tag to remove, specified as a string or Urn object. + + Returns: + The DataJobPatchBuilder instance. + """ + if isinstance(tag, str) and not tag.startswith("urn:li:tag:"): + tag = TagUrn.create_from_id(tag) + self._add_patch(GlobalTags.ASPECT_NAME, "remove", path=f"/tags/{tag}", value={}) + return self + + def add_term(self, term: Term) -> "DataJobPatchBuilder": + """ + Adds a glossary term to the DataJobPatchBuilder. + + Args: + term: The Term object representing the glossary term to be added. + + Returns: + The DataJobPatchBuilder instance. + """ + self._add_patch( + GlossaryTerms.ASPECT_NAME, "add", path=f"/terms/{term.urn}", value=term + ) + return self + + def remove_term(self, term: Union[str, Urn]) -> "DataJobPatchBuilder": + """ + Removes a glossary term from the DataJobPatchBuilder. + + Args: + term: The term to remove, specified as a string or Urn object. + + Returns: + The DataJobPatchBuilder instance. + """ + if isinstance(term, str) and not term.startswith("urn:li:glossaryTerm:"): + term = "urn:li:glossaryTerm:" + term + self._add_patch( + GlossaryTerms.ASPECT_NAME, "remove", path=f"/terms/{term}", value={} + ) + return self + + def set_custom_properties( + self, custom_properties: Dict[str, str] + ) -> "DataJobPatchBuilder": + """ + Sets the custom properties for the DataJobPatchBuilder. + + Args: + custom_properties: A dictionary containing the custom properties to be set. + + Returns: + The DataJobPatchBuilder instance. + + Notes: + This method replaces all existing custom properties with the given dictionary. + """ + self._add_patch( + DataJobInfo.ASPECT_NAME, + "replace", + path="/customProperties", + value=custom_properties, + ) + return self + + def add_custom_property(self, key: str, value: str) -> "DataJobPatchBuilder": + """ + Adds a custom property to the DataJobPatchBuilder. + + Args: + key: The key of the custom property. + value: The value of the custom property. + + Returns: + The DataJobPatchBuilder instance. + """ + self.custom_properties_patch_helper.add_property(key, value) + return self + + def remove_custom_property(self, key: str) -> "DataJobPatchBuilder": + """ + Removes a custom property from the DataJobPatchBuilder. + + Args: + key: The key of the custom property to remove. + + Returns: + The DataJobPatchBuilder instance. + """ + self.custom_properties_patch_helper.remove_property(key) + return self diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/datajob/DataJobInputOutputPatchBuilder.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/datajob/DataJobInputOutputPatchBuilder.java new file mode 100644 index 0000000000000..1ff6e817e40cf --- /dev/null +++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/datajob/DataJobInputOutputPatchBuilder.java @@ -0,0 +1,214 @@ +package datahub.client.patch.datajob; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.linkedin.common.Edge; +import com.linkedin.common.urn.DataJobUrn; +import com.linkedin.common.urn.DatasetUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.metadata.graph.LineageDirection; +import datahub.client.patch.AbstractMultiFieldPatchBuilder; +import datahub.client.patch.PatchOperationType; +import javax.annotation.Nonnull; +import org.apache.commons.lang3.tuple.ImmutableTriple; + +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*; +import static com.linkedin.metadata.Constants.*; + + +public class DataJobInputOutputPatchBuilder extends AbstractMultiFieldPatchBuilder { + private static final String INPUT_DATA_JOB_EDGES_PATH_START = "/inputDatajobEdges/"; + private static final String INPUT_DATASET_EDGES_PATH_START = "/inputDatasetEdges/"; + private static final String OUTPUT_DATASET_EDGES_PATH_START = "/outputDatasetEdges/"; + + private static final String DESTINATION_URN_KEY = "destinationUrn"; + private static final String SOURCE_URN_KEY = "sourceUrn"; + private static final String LAST_MODIFIED_KEY = "lastModified"; + private static final String CREATED_KEY = "created"; + private static final String PROPERTIES_KEY = "properties"; + + private static final String INPUT_DATASET_FIELDS_PATH_START = "/inputDatasetFields/"; + private static final String OUTPUT_DATASET_FIELDS_PATH_START = "/outputDatasetFields/"; + + private static final String TIME_KEY = "time"; + private static final String ACTOR_KEY = "actor"; + private static final String IMPERSONATOR_KEY = "impersonator"; + private static final String MESSAGE_KEY = "message"; + + // Simplified with just Urn + public DataJobInputOutputPatchBuilder addInputDatajobEdge(@Nonnull DataJobUrn dataJobUrn) { + ObjectNode value = createEdgeValue(dataJobUrn); + + pathValues.add(ImmutableTriple.of(PatchOperationType.ADD.getValue(), INPUT_DATA_JOB_EDGES_PATH_START + dataJobUrn, value)); + return this; + } + + public DataJobInputOutputPatchBuilder removeInputDatajobEdge(@Nonnull DataJobUrn dataJobUrn) { + pathValues.add(ImmutableTriple.of(PatchOperationType.REMOVE.getValue(), INPUT_DATA_JOB_EDGES_PATH_START + dataJobUrn, null)); + return this; + } + + public DataJobInputOutputPatchBuilder addInputDatasetEdge(@Nonnull DatasetUrn datasetUrn) { + ObjectNode value = createEdgeValue(datasetUrn); + + pathValues.add(ImmutableTriple.of(PatchOperationType.ADD.getValue(), INPUT_DATASET_EDGES_PATH_START + datasetUrn, value)); + return this; + } + + public DataJobInputOutputPatchBuilder removeInputDatasetEdge(@Nonnull DatasetUrn datasetUrn) { + pathValues.add(ImmutableTriple.of(PatchOperationType.REMOVE.getValue(), INPUT_DATASET_EDGES_PATH_START + datasetUrn, null)); + return this; + } + + public DataJobInputOutputPatchBuilder addOutputDatasetEdge(@Nonnull DatasetUrn datasetUrn) { + ObjectNode value = createEdgeValue(datasetUrn); + + pathValues.add(ImmutableTriple.of(PatchOperationType.ADD.getValue(), OUTPUT_DATASET_EDGES_PATH_START + datasetUrn, value)); + return this; + } + + public DataJobInputOutputPatchBuilder removeOutputDatasetEdge(@Nonnull DatasetUrn datasetUrn) { + pathValues.add(ImmutableTriple.of(PatchOperationType.REMOVE.getValue(), OUTPUT_DATASET_EDGES_PATH_START + datasetUrn, null)); + return this; + } + + public DataJobInputOutputPatchBuilder addInputDatasetField(@Nonnull Urn urn) { + TextNode textNode = instance.textNode(urn.toString()); + pathValues.add(ImmutableTriple.of(PatchOperationType.ADD.getValue(), INPUT_DATASET_FIELDS_PATH_START + urn, textNode)); + + return this; + } + + public DataJobInputOutputPatchBuilder removeInputDatasetField(@Nonnull Urn urn) { + pathValues.add(ImmutableTriple.of(PatchOperationType.REMOVE.getValue(), INPUT_DATASET_FIELDS_PATH_START + urn, null)); + return this; + } + + public DataJobInputOutputPatchBuilder addOutputDatasetField(@Nonnull Urn urn) { + TextNode textNode = instance.textNode(urn.toString()); + pathValues.add(ImmutableTriple.of(PatchOperationType.ADD.getValue(), OUTPUT_DATASET_FIELDS_PATH_START + urn, textNode)); + + return this; + } + + public DataJobInputOutputPatchBuilder removeOutputDatasetField(@Nonnull Urn urn) { + pathValues.add(ImmutableTriple.of(PatchOperationType.REMOVE.getValue(), OUTPUT_DATASET_FIELDS_PATH_START + urn, null)); + return this; + } + + // Full Edge modification + public DataJobInputOutputPatchBuilder addEdge(@Nonnull Edge edge, @Nonnull LineageDirection direction) { + ObjectNode value = createEdgeValue(edge); + String path = getEdgePath(edge, direction); + + pathValues.add(ImmutableTriple.of(PatchOperationType.ADD.getValue(), path, value)); + return this; + } + + public DataJobInputOutputPatchBuilder removeEdge(@Nonnull Edge edge, @Nonnull LineageDirection direction) { + String path = getEdgePath(edge, direction); + + pathValues.add(ImmutableTriple.of(PatchOperationType.REMOVE.getValue(), path, null)); + return this; + } + + private ObjectNode createEdgeValue(@Nonnull Urn urn) { + ObjectNode value = instance.objectNode(); + ObjectNode auditStamp = instance.objectNode(); + auditStamp.put(TIME_KEY, System.currentTimeMillis()) + .put(ACTOR_KEY, UNKNOWN_ACTOR); + + value.put(DESTINATION_URN_KEY, urn.toString()) + .set(LAST_MODIFIED_KEY, auditStamp); + value.set(CREATED_KEY, auditStamp); + + return value; + } + + private ObjectNode createEdgeValue(@Nonnull Edge edge) { + ObjectNode value = instance.objectNode(); + + ObjectNode created = instance.objectNode(); + if (edge.getCreated() == null) { + created.put(TIME_KEY, System.currentTimeMillis()).put(ACTOR_KEY, UNKNOWN_ACTOR); + } else { + created + .put(TIME_KEY, edge.getCreated().getTime()) + .put(ACTOR_KEY, edge.getCreated().getActor().toString()); + if (edge.getCreated().getImpersonator() != null) { + created.put(IMPERSONATOR_KEY, edge.getCreated().getImpersonator().toString()); + } + if (edge.getCreated().getMessage() != null) { + created.put(MESSAGE_KEY, edge.getCreated().getMessage()); + } + } + value.set(CREATED_KEY, created); + + ObjectNode lastModified = instance.objectNode(); + if (edge.getLastModified() == null) { + lastModified.put(TIME_KEY, System.currentTimeMillis()).put(ACTOR_KEY, UNKNOWN_ACTOR); + } else { + lastModified + .put(TIME_KEY, edge.getLastModified().getTime()) + .put(ACTOR_KEY, edge.getLastModified().getActor().toString()); + if (edge.getLastModified() .getImpersonator() != null) { + lastModified.put(IMPERSONATOR_KEY, edge.getLastModified().getImpersonator().toString()); + } + if (edge.getLastModified() .getMessage() != null) { + lastModified.put(MESSAGE_KEY, edge.getLastModified() .getMessage()); + } + } + value.set(LAST_MODIFIED_KEY, lastModified); + + if (edge.getProperties() != null) { + ObjectNode propertiesNode = instance.objectNode(); + edge.getProperties().forEach((k, v) -> propertiesNode.set(k, instance.textNode(v))); + value.set(PROPERTIES_KEY, propertiesNode); + } + + value.put(DESTINATION_URN_KEY, edge.getDestinationUrn().toString()); + if (edge.getSourceUrn() != null) { + value.put(SOURCE_URN_KEY, edge.getSourceUrn().toString()); + } + + + return value; + } + + /** + * Determines Edge path based on supplied Urn, if not a valid entity type throws IllegalArgumentException + * @param edge + * @return + * @throws IllegalArgumentException if destinationUrn is an invalid entity type + */ + private String getEdgePath(@Nonnull Edge edge, LineageDirection direction) { + Urn destinationUrn = edge.getDestinationUrn(); + + if (DATASET_ENTITY_NAME.equals(destinationUrn.getEntityType()) && LineageDirection.UPSTREAM.equals(direction)) { + return INPUT_DATASET_EDGES_PATH_START + destinationUrn; + } + + if (DATASET_ENTITY_NAME.equals(destinationUrn.getEntityType()) && LineageDirection.DOWNSTREAM.equals(direction)) { + return INPUT_DATASET_EDGES_PATH_START + destinationUrn; + } + + if (DATA_JOB_ENTITY_NAME.equals(destinationUrn.getEntityType()) && LineageDirection.UPSTREAM.equals(direction)) { + return INPUT_DATA_JOB_EDGES_PATH_START + destinationUrn; + } + + // TODO: Output Data Jobs not supported by aspect, add here if this changes + + throw new IllegalArgumentException(String.format("Unsupported entity type: %s", destinationUrn.getEntityType())); + } + + @Override + protected String getAspectName() { + return DATA_JOB_INPUT_OUTPUT_ASPECT_NAME; + } + + @Override + protected String getEntityType() { + return DATA_JOB_ENTITY_NAME; + } + +} diff --git a/metadata-integration/java/datahub-client/src/test/java/datahub/client/patch/PatchTest.java b/metadata-integration/java/datahub-client/src/test/java/datahub/client/patch/PatchTest.java index bf84c66ad0f3d..c90d3f0d2179e 100644 --- a/metadata-integration/java/datahub-client/src/test/java/datahub/client/patch/PatchTest.java +++ b/metadata-integration/java/datahub-client/src/test/java/datahub/client/patch/PatchTest.java @@ -1,16 +1,20 @@ package datahub.client.patch; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.Edge; import com.linkedin.common.FabricType; import com.linkedin.common.GlossaryTermAssociation; import com.linkedin.common.OwnershipType; import com.linkedin.common.TagAssociation; import com.linkedin.common.urn.CorpuserUrn; +import com.linkedin.common.urn.DataJobUrn; import com.linkedin.common.urn.DataPlatformUrn; import com.linkedin.common.urn.DatasetUrn; import com.linkedin.common.urn.GlossaryTermUrn; import com.linkedin.common.urn.TagUrn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.dataset.DatasetLineageType; +import com.linkedin.metadata.graph.LineageDirection; import com.linkedin.mxe.MetadataChangeProposal; import datahub.client.MetadataWriteResponse; import datahub.client.file.FileEmitter; @@ -18,6 +22,7 @@ import datahub.client.patch.common.OwnershipPatchBuilder; import datahub.client.patch.dataflow.DataFlowInfoPatchBuilder; import datahub.client.patch.datajob.DataJobInfoPatchBuilder; +import datahub.client.patch.datajob.DataJobInputOutputPatchBuilder; import datahub.client.patch.dataset.DatasetPropertiesPatchBuilder; import datahub.client.patch.dataset.EditableSchemaMetadataPatchBuilder; import datahub.client.patch.dataset.UpstreamLineagePatchBuilder; @@ -33,6 +38,8 @@ import org.junit.runner.RunWith; import org.mockito.junit.MockitoJUnitRunner; +import static com.linkedin.metadata.Constants.*; + @RunWith(MockitoJUnitRunner.class) public class PatchTest { @@ -364,4 +371,80 @@ public void testLocalDataFlowInfoRemove() { } } + @Test + @Ignore + public void testLocalDataJobInputAdd() { + RestEmitter restEmitter = new RestEmitter(RestEmitterConfig.builder().build()); + try { + MetadataChangeProposal dataJobIOPatch = new DataJobInputOutputPatchBuilder() + .urn(UrnUtils.getUrn("urn:li:dataJob:(urn:li:dataFlow:(orchestrator,flowId,cluster),jobId)")) + .addInputDatasetEdge(DatasetUrn.createFromString("urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)")) + .addOutputDatasetEdge(DatasetUrn.createFromString("urn:li:dataset:(urn:li:dataPlatform:kafka,SampleHiveDataset,PROD)")) + .addInputDatajobEdge(DataJobUrn.createFromString("urn:li:dataJob:(urn:li:dataFlow:(orchestrator,flowId,cluster),jobId2)")) + .addInputDatasetField(UrnUtils.getUrn("urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)")) + .addOutputDatasetField(UrnUtils.getUrn("urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)")) + .build(); + Future response = restEmitter.emit(dataJobIOPatch); + + System.out.println(response.get().getResponseContent()); + + } catch (URISyntaxException | IOException | ExecutionException | InterruptedException e) { + System.out.println(Arrays.asList(e.getStackTrace())); + } + } + + @Test + @Ignore + public void testLocalDataJobInputRemove() { + RestEmitter restEmitter = new RestEmitter(RestEmitterConfig.builder().build()); + try { + MetadataChangeProposal dataJobIOPatch = new DataJobInputOutputPatchBuilder() + .urn(UrnUtils.getUrn("urn:li:dataJob:(urn:li:dataFlow:(orchestrator,flowId,cluster),jobId)")) + .removeInputDatasetEdge(DatasetUrn.createFromString("urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)")) + .removeOutputDatasetEdge(DatasetUrn.createFromString("urn:li:dataset:(urn:li:dataPlatform:kafka,SampleHiveDataset,PROD)")) + .removeInputDatajobEdge(DataJobUrn.createFromString("urn:li:dataJob:(urn:li:dataFlow:(orchestrator,flowId,cluster),jobId2)")) + .removeInputDatasetField(UrnUtils.getUrn("urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)")) + .removeOutputDatasetField(UrnUtils.getUrn("urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)")) + .build(); + Future response = restEmitter.emit(dataJobIOPatch); + + System.out.println(response.get().getResponseContent()); + + } catch (URISyntaxException | IOException | ExecutionException | InterruptedException e) { + System.out.println(Arrays.asList(e.getStackTrace())); + } + } + + @Test + @Ignore + public void testLocalDataJobInputAddEdge() { + RestEmitter restEmitter = new RestEmitter(RestEmitterConfig.builder().build()); + try { + Edge inputDataset = new Edge() + .setDestinationUrn(DatasetUrn.createFromString("urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)")) + .setCreated(new AuditStamp().setTime(System.currentTimeMillis()).setActor(UrnUtils.getUrn(UNKNOWN_ACTOR))) + .setLastModified(new AuditStamp().setTime(System.currentTimeMillis()).setActor(UrnUtils.getUrn(UNKNOWN_ACTOR))); + Edge outputDataset = new Edge() + .setDestinationUrn(DatasetUrn.createFromString("urn:li:dataset:(urn:li:dataPlatform:kafka,SampleHiveDataset,PROD)")) + .setCreated(new AuditStamp().setTime(System.currentTimeMillis()).setActor(UrnUtils.getUrn(UNKNOWN_ACTOR))) + .setLastModified(new AuditStamp().setTime(System.currentTimeMillis()).setActor(UrnUtils.getUrn(UNKNOWN_ACTOR))); + Edge inputDataJob = new Edge() + .setDestinationUrn(DataJobUrn.createFromString("urn:li:dataJob:(urn:li:dataFlow:(orchestrator,flowId,cluster),jobId2)")) + .setCreated(new AuditStamp().setTime(System.currentTimeMillis()).setActor(UrnUtils.getUrn(UNKNOWN_ACTOR))) + .setLastModified(new AuditStamp().setTime(System.currentTimeMillis()).setActor(UrnUtils.getUrn(UNKNOWN_ACTOR))); + MetadataChangeProposal dataJobIOPatch = new DataJobInputOutputPatchBuilder() + .urn(UrnUtils.getUrn("urn:li:dataJob:(urn:li:dataFlow:(orchestrator,flowId,cluster),jobId)")) + .addEdge(inputDataset, LineageDirection.UPSTREAM) + .addEdge(outputDataset, LineageDirection.DOWNSTREAM) + .addEdge(inputDataJob, LineageDirection.UPSTREAM) + .build(); + Future response = restEmitter.emit(dataJobIOPatch); + + System.out.println(response.get().getResponseContent()); + + } catch (URISyntaxException | IOException | ExecutionException | InterruptedException e) { + System.out.println(Arrays.asList(e.getStackTrace())); + } + } + } diff --git a/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DataJobLineageAdd.java b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DataJobLineageAdd.java new file mode 100644 index 0000000000000..4fd5c771caeba --- /dev/null +++ b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DataJobLineageAdd.java @@ -0,0 +1,59 @@ +package io.datahubproject.examples; + +import com.linkedin.common.urn.DataJobUrn; +import com.linkedin.common.urn.DatasetUrn; +import com.linkedin.common.urn.UrnUtils; +import datahub.client.MetadataWriteResponse; +import datahub.client.patch.datajob.DataJobInputOutputPatchBuilder; +import datahub.client.rest.RestEmitter; +import java.io.IOException; +import com.linkedin.mxe.MetadataChangeProposal; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import lombok.extern.slf4j.Slf4j; + + +@Slf4j +class DataJobLineageAdd { + + private DataJobLineageAdd() { + + } + + /** + * Adds lineage to an existing DataJob without affecting any lineage + * @param args + * @throws IOException + * @throws ExecutionException + * @throws InterruptedException + */ + public static void main(String[] args) throws IOException, ExecutionException, InterruptedException { + String token = ""; + try (RestEmitter emitter = RestEmitter.create( + b -> b.server("http://localhost:8080") + .token(token) + )) { + MetadataChangeProposal dataJobIOPatch = new DataJobInputOutputPatchBuilder().urn(UrnUtils + .getUrn("urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_456)")) + .addInputDatasetEdge(DatasetUrn.createFromString("urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)")) + .addOutputDatasetEdge(DatasetUrn.createFromString("urn:li:dataset:(urn:li:dataPlatform:kafka,SampleHiveDataset,PROD)")) + .addInputDatajobEdge(DataJobUrn.createFromString("urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_123)")) + .addInputDatasetField(UrnUtils.getUrn( + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)")) + .addOutputDatasetField(UrnUtils.getUrn( + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)")) + .build(); + + Future response = emitter.emit(dataJobIOPatch); + + System.out.println(response.get().getResponseContent()); + } catch (Exception e) { + log.error("Failed to emit metadata to DataHub", e); + throw new RuntimeException(e); + } + + } + +} + + diff --git a/smoke-test/tests/patch/common_patch_tests.py b/smoke-test/tests/patch/common_patch_tests.py new file mode 100644 index 0000000000000..574e4fd4e4c88 --- /dev/null +++ b/smoke-test/tests/patch/common_patch_tests.py @@ -0,0 +1,277 @@ +import time +import uuid +from typing import Dict, Optional, Type + +from datahub.emitter.mce_builder import ( + make_tag_urn, + make_term_urn, + make_user_urn, +) +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.mcp_patch_builder import MetadataPatchProposal +from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig +from datahub.metadata.schema_classes import ( + AuditStampClass, + GlobalTagsClass, + GlossaryTermAssociationClass, + GlossaryTermsClass, + OwnerClass, + OwnershipClass, + OwnershipTypeClass, + TagAssociationClass, + _Aspect, +) + + +def helper_test_entity_terms_patch( + test_entity_urn: str, patch_builder_class: Type[MetadataPatchProposal] +): + def get_terms(graph, entity_urn): + return graph.get_aspect( + entity_urn=entity_urn, + aspect_type=GlossaryTermsClass, + ) + + term_urn = make_term_urn(term=f"testTerm-{uuid.uuid4()}") + + term_association = GlossaryTermAssociationClass( + urn=term_urn, context="test" + ) + global_terms = GlossaryTermsClass( + terms=[term_association], + auditStamp=AuditStampClass( + time=int(time.time() * 1000.0), actor=make_user_urn("tester") + ), + ) + mcpw = MetadataChangeProposalWrapper( + entityUrn=test_entity_urn, aspect=global_terms + ) + + with DataHubGraph(DataHubGraphConfig()) as graph: + graph.emit_mcp(mcpw) + terms_read = get_terms(graph, test_entity_urn) + assert terms_read.terms[0].urn == term_urn + assert terms_read.terms[0].context == "test" + + new_term = GlossaryTermAssociationClass( + urn=make_term_urn(f"test-{uuid.uuid4()}") + ) + patch_builder = patch_builder_class(test_entity_urn) + assert hasattr(patch_builder, "add_term") + for patch_mcp in patch_builder.add_term(new_term).build(): + graph.emit_mcp(patch_mcp) + pass + + terms_read = get_terms(graph, test_entity_urn) + + assert terms_read.terms[0].urn == term_urn + assert terms_read.terms[0].context == "test" + assert terms_read.terms[1].urn == new_term.urn + assert terms_read.terms[1].context is None + + for patch_mcp in ( + patch_builder_class(test_entity_urn).remove_term(term_urn).build() + ): + graph.emit_mcp(patch_mcp) + pass + + terms_read = get_terms(graph, test_entity_urn) + assert len(terms_read.terms) == 1 + assert terms_read.terms[0].urn == new_term.urn + + +def helper_test_dataset_tags_patch( + test_entity_urn: str, patch_builder_class: Type[MetadataPatchProposal] +): + + tag_urn = make_tag_urn(tag=f"testTag-{uuid.uuid4()}") + + tag_association = TagAssociationClass(tag=tag_urn, context="test") + global_tags = GlobalTagsClass(tags=[tag_association]) + mcpw = MetadataChangeProposalWrapper( + entityUrn=test_entity_urn, aspect=global_tags + ) + + with DataHubGraph(DataHubGraphConfig()) as graph: + graph.emit_mcp(mcpw) + tags_read: GlobalTagsClass = graph.get_aspect( + entity_urn=test_entity_urn, + aspect_type=GlobalTagsClass, + ) + assert tags_read.tags[0].tag == tag_urn + assert tags_read.tags[0].context == "test" + + new_tag = TagAssociationClass(tag=make_tag_urn(f"test-{uuid.uuid4()}")) + patch_builder = patch_builder_class(test_entity_urn) + assert hasattr(patch_builder, "add_tag") + for patch_mcp in patch_builder.add_tag(new_tag).build(): + graph.emit_mcp(patch_mcp) + pass + + tags_read = graph.get_aspect( + entity_urn=test_entity_urn, + aspect_type=GlobalTagsClass, + ) + assert tags_read.tags[0].tag == tag_urn + assert tags_read.tags[0].context == "test" + assert tags_read.tags[1].tag == new_tag.tag + assert tags_read.tags[1].context is None + + for patch_mcp in ( + patch_builder_class(test_entity_urn).remove_tag(tag_urn).build() + ): + graph.emit_mcp(patch_mcp) + pass + + tags_read = graph.get_aspect( + entity_urn=test_entity_urn, + aspect_type=GlobalTagsClass, + ) + assert len(tags_read.tags) == 1 + assert tags_read.tags[0].tag == new_tag.tag + + +def helper_test_ownership_patch( + test_entity_urn: str, patch_builder_class: Type[MetadataPatchProposal] +): + owner_to_set = OwnerClass( + owner=make_user_urn("jdoe"), type=OwnershipTypeClass.DATAOWNER + ) + ownership_to_set = OwnershipClass(owners=[owner_to_set]) + + owner_to_add = OwnerClass( + owner=make_user_urn("gdoe"), type=OwnershipTypeClass.DATAOWNER + ) + mcpw = MetadataChangeProposalWrapper( + entityUrn=test_entity_urn, aspect=ownership_to_set + ) + with DataHubGraph(DataHubGraphConfig()) as graph: + graph.emit_mcp(mcpw) + owner: OwnershipClass = graph.get_aspect( + entity_urn=test_entity_urn, aspect_type=OwnershipClass + ) + assert owner.owners[0].owner == make_user_urn("jdoe") + + for patch_mcp in ( + patch_builder_class(test_entity_urn) + .add_owner(owner_to_add) + .build() + ): + graph.emit_mcp(patch_mcp) + + owner = graph.get_aspect( + entity_urn=test_entity_urn, aspect_type=OwnershipClass + ) + assert len(owner.owners) == 2 + + for patch_mcp in ( + patch_builder_class(test_entity_urn) + .remove_owner(make_user_urn("gdoe")) + .build() + ): + graph.emit_mcp(patch_mcp) + + owner = graph.get_aspect( + entity_urn=test_entity_urn, aspect_type=OwnershipClass + ) + assert len(owner.owners) == 1 + assert owner.owners[0].owner == make_user_urn("jdoe") + + +def helper_test_custom_properties_patch( + test_entity_urn: str, + patch_builder_class: Type[MetadataPatchProposal], + custom_properties_aspect_class: Type[_Aspect], + base_aspect: _Aspect, +): + def get_custom_properties( + graph: DataHubGraph, entity_urn: str + ) -> Optional[Dict[str, str]]: + custom_properties_aspect = graph.get_aspect( + entity_urn=entity_urn, + aspect_type=custom_properties_aspect_class, + ) + assert custom_properties_aspect + return custom_properties_aspect.customProperties + + base_property_map = {"base_property": "base_property_value"} + + orig_aspect = base_aspect + assert hasattr(orig_aspect, "customProperties") + orig_aspect.customProperties = base_property_map + mcpw = MetadataChangeProposalWrapper( + entityUrn=test_entity_urn, aspect=orig_aspect + ) + + with DataHubGraph(DataHubGraphConfig()) as graph: + graph.emit(mcpw) + # assert custom properties looks as expected + custom_properties = get_custom_properties(graph, test_entity_urn) + assert custom_properties + for k, v in base_property_map.items(): + assert custom_properties[k] == v + + new_properties = { + "test_property": "test_value", + "test_property1": "test_value1", + } + + entity_patch_builder = patch_builder_class(test_entity_urn) + for k, v in new_properties.items(): + entity_patch_builder.add_custom_property(k, v) + + for patch_mcp in entity_patch_builder.build(): + graph.emit_mcp(patch_mcp) + + custom_properties = get_custom_properties(graph, test_entity_urn) + + assert custom_properties is not None + for k, v in new_properties.items(): + assert custom_properties[k] == v + + # ensure exising properties were not touched + for k, v in base_property_map.items(): + assert custom_properties[k] == v + + # Remove property + for patch_mcp in ( + patch_builder_class(test_entity_urn) + .remove_custom_property("test_property") + .build() + ): + graph.emit_mcp(patch_mcp) + + custom_properties = get_custom_properties(graph, test_entity_urn) + + assert custom_properties is not None + assert "test_property" not in custom_properties + assert custom_properties["test_property1"] == "test_value1" + + # ensure exising properties were not touched + for k, v in base_property_map.items(): + assert custom_properties[k] == v + + # Replace custom properties + for patch_mcp in ( + patch_builder_class(test_entity_urn) + .set_custom_properties(new_properties) + .build() + ): + graph.emit_mcp(patch_mcp) + + custom_properties = get_custom_properties(graph, test_entity_urn) + + assert custom_properties is not None + for k in base_property_map: + assert k not in custom_properties + for k, v in new_properties.items(): + assert custom_properties[k] == v + + # ensure existing fields were not touched + full_aspect: Optional[_Aspect] = graph.get_aspect( + test_entity_urn, custom_properties_aspect_class + ) + + assert full_aspect + for k, v in orig_aspect.__dict__.items(): + assert full_aspect.__dict__[k] == v diff --git a/smoke-test/tests/patch/test_datajob_patches.py b/smoke-test/tests/patch/test_datajob_patches.py new file mode 100644 index 0000000000000..407410ee89914 --- /dev/null +++ b/smoke-test/tests/patch/test_datajob_patches.py @@ -0,0 +1,135 @@ +import uuid + +from datahub.emitter.mce_builder import make_data_job_urn, make_dataset_urn +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig +from datahub.metadata.schema_classes import ( + DataJobInfoClass, + DataJobInputOutputClass, + EdgeClass, +) +from datahub.specific.datajob import DataJobPatchBuilder + +from tests.patch.common_patch_tests import ( + helper_test_custom_properties_patch, + helper_test_dataset_tags_patch, + helper_test_entity_terms_patch, + helper_test_ownership_patch, +) + + +def _make_test_datajob_urn( + seedFlow: str = "SampleAirflowDag", seedTask: str = "SampleAirflowTask" +): + return make_data_job_urn( + orchestrator="airflow", + flow_id=f"{seedFlow}{uuid.uuid4()}", + job_id=f"{seedTask}{uuid.uuid4()}", + ) + + +# Common Aspect Patch Tests +# Ownership +def test_datajob_ownership_patch(wait_for_healthchecks): + datajob_urn = _make_test_datajob_urn() + helper_test_ownership_patch(datajob_urn, DataJobPatchBuilder) + + +# Tags +def test_datajob_tags_patch(wait_for_healthchecks): + helper_test_dataset_tags_patch( + _make_test_datajob_urn(), DataJobPatchBuilder + ) + + +# Terms +def test_dataset_terms_patch(wait_for_healthchecks): + helper_test_entity_terms_patch( + _make_test_datajob_urn(), DataJobPatchBuilder + ) + + +# Custom Properties +def test_custom_properties_patch(wait_for_healthchecks): + orig_datajob_info = DataJobInfoClass(name="test_name", type="TestJobType") + helper_test_custom_properties_patch( + test_entity_urn=_make_test_datajob_urn(), + patch_builder_class=DataJobPatchBuilder, + custom_properties_aspect_class=DataJobInfoClass, + base_aspect=orig_datajob_info, + ) + + +# Specific Aspect Patch Tests +# Input/Output +def test_datajob_inputoutput_dataset_patch(wait_for_healthchecks): + datajob_urn = _make_test_datajob_urn() + + other_dataset_urn = make_dataset_urn( + platform="hive", name=f"SampleHiveDataset2-{uuid.uuid4()}", env="PROD" + ) + + patch_dataset_urn = make_dataset_urn( + platform="hive", name=f"SampleHiveDataset3-{uuid.uuid4()}", env="PROD" + ) + + inputoutput_lineage = DataJobInputOutputClass( + inputDatasets=[], + outputDatasets=[], + inputDatasetEdges=[EdgeClass(destinationUrn=other_dataset_urn)], + ) + dataset_input_lineage_to_add = EdgeClass(destinationUrn=patch_dataset_urn) + mcpw = MetadataChangeProposalWrapper( + entityUrn=datajob_urn, aspect=inputoutput_lineage + ) + + with DataHubGraph(DataHubGraphConfig()) as graph: + graph.emit_mcp(mcpw) + inputoutput_lineage_read: DataJobInputOutputClass = graph.get_aspect( + entity_urn=datajob_urn, + aspect_type=DataJobInputOutputClass, + ) + assert ( + inputoutput_lineage_read.inputDatasetEdges[0].destinationUrn + == other_dataset_urn + ) + + for patch_mcp in ( + DataJobPatchBuilder(datajob_urn) + .add_input_dataset(dataset_input_lineage_to_add) + .build() + ): + graph.emit_mcp(patch_mcp) + pass + + inputoutput_lineage_read = graph.get_aspect( + entity_urn=datajob_urn, + aspect_type=DataJobInputOutputClass, + ) + assert len(inputoutput_lineage_read.inputDatasetEdges) == 2 + assert ( + inputoutput_lineage_read.inputDatasetEdges[0].destinationUrn + == other_dataset_urn + ) + assert ( + inputoutput_lineage_read.inputDatasetEdges[1].destinationUrn + == patch_dataset_urn + ) + + for patch_mcp in ( + DataJobPatchBuilder(datajob_urn) + .remove_input_dataset(patch_dataset_urn) + .build() + ): + graph.emit_mcp(patch_mcp) + pass + + inputoutput_lineage_read = graph.get_aspect( + entity_urn=datajob_urn, + aspect_type=DataJobInputOutputClass, + ) + assert len(inputoutput_lineage_read.inputDatasetEdges) == 1 + assert ( + inputoutput_lineage_read.inputDatasetEdges[0].destinationUrn + == other_dataset_urn + ) diff --git a/smoke-test/tests/patch/test_dataset_patches.py b/smoke-test/tests/patch/test_dataset_patches.py index b4ada2094a8f3..239aab64675d8 100644 --- a/smoke-test/tests/patch/test_dataset_patches.py +++ b/smoke-test/tests/patch/test_dataset_patches.py @@ -20,48 +20,30 @@ UpstreamClass, UpstreamLineageClass) from datahub.specific.dataset import DatasetPatchBuilder +from tests.patch.common_patch_tests import helper_test_entity_terms_patch, helper_test_dataset_tags_patch, helper_test_ownership_patch, helper_test_custom_properties_patch +# Common Aspect Patch Tests +# Ownership def test_dataset_ownership_patch(wait_for_healthchecks): dataset_urn = make_dataset_urn( platform="hive", name=f"SampleHiveDataset{uuid.uuid4()}", env="PROD" ) - owner_to_set = OwnerClass( - owner=make_user_urn("jdoe"), type=OwnershipTypeClass.DATAOWNER - ) - ownership_to_set = OwnershipClass(owners=[owner_to_set]) + helper_test_ownership_patch(dataset_urn, DatasetPatchBuilder) - owner_to_add = OwnerClass( - owner=make_user_urn("gdoe"), type=OwnershipTypeClass.DATAOWNER +# Tags +def test_dataset_tags_patch(wait_for_healthchecks): + dataset_urn = make_dataset_urn( + platform="hive", name=f"SampleHiveDataset-{uuid.uuid4()}", env="PROD" ) - mcpw = MetadataChangeProposalWrapper(entityUrn=dataset_urn, aspect=ownership_to_set) - with DataHubGraph(DataHubGraphConfig()) as graph: - graph.emit_mcp(mcpw) - owner = graph.get_aspect_v2( - entity_urn=dataset_urn, aspect_type=OwnershipClass, aspect="ownership" - ) - assert owner.owners[0].owner == make_user_urn("jdoe") - - for patch_mcp in ( - DatasetPatchBuilder(dataset_urn).add_owner(owner_to_add).build() - ): - graph.emit_mcp(patch_mcp) - - owner = graph.get_aspect_v2( - entity_urn=dataset_urn, aspect_type=OwnershipClass, aspect="ownership" - ) - assert len(owner.owners) == 2 - - for patch_mcp in ( - DatasetPatchBuilder(dataset_urn).remove_owner(make_user_urn("gdoe")).build() - ): - graph.emit_mcp(patch_mcp) + helper_test_dataset_tags_patch(dataset_urn, DatasetPatchBuilder) - owner = graph.get_aspect_v2( - entity_urn=dataset_urn, aspect_type=OwnershipClass, aspect="ownership" - ) - assert len(owner.owners) == 1 - assert owner.owners[0].owner == make_user_urn("jdoe") +# Terms +def test_dataset_terms_patch(wait_for_healthchecks): + dataset_urn = make_dataset_urn( + platform="hive", name=f"SampleHiveDataset-{uuid.uuid4()}", env="PROD" + ) + helper_test_entity_terms_patch(dataset_urn, DatasetPatchBuilder) def test_dataset_upstream_lineage_patch(wait_for_healthchecks): @@ -130,107 +112,6 @@ def test_dataset_upstream_lineage_patch(wait_for_healthchecks): assert upstream_lineage_read.upstreams[0].dataset == other_dataset_urn -def test_dataset_tags_patch(wait_for_healthchecks): - dataset_urn = make_dataset_urn( - platform="hive", name=f"SampleHiveDataset-{uuid.uuid4()}", env="PROD" - ) - - tag_urn = make_tag_urn(tag=f"testTag-{uuid.uuid4()}") - - tag_association = TagAssociationClass(tag=tag_urn, context="test") - global_tags = GlobalTagsClass(tags=[tag_association]) - mcpw = MetadataChangeProposalWrapper(entityUrn=dataset_urn, aspect=global_tags) - - with DataHubGraph(DataHubGraphConfig()) as graph: - graph.emit_mcp(mcpw) - tags_read = graph.get_aspect_v2( - entity_urn=dataset_urn, - aspect_type=GlobalTagsClass, - aspect="globalTags", - ) - assert tags_read.tags[0].tag == tag_urn - assert tags_read.tags[0].context == "test" - - new_tag = TagAssociationClass(tag=make_tag_urn(f"test-{uuid.uuid4()}")) - for patch_mcp in DatasetPatchBuilder(dataset_urn).add_tag(new_tag).build(): - graph.emit_mcp(patch_mcp) - pass - - tags_read = graph.get_aspect_v2( - entity_urn=dataset_urn, - aspect_type=GlobalTagsClass, - aspect="globalTags", - ) - assert tags_read.tags[0].tag == tag_urn - assert tags_read.tags[0].context == "test" - assert tags_read.tags[1].tag == new_tag.tag - assert tags_read.tags[1].context is None - - for patch_mcp in DatasetPatchBuilder(dataset_urn).remove_tag(tag_urn).build(): - graph.emit_mcp(patch_mcp) - pass - - tags_read = graph.get_aspect_v2( - entity_urn=dataset_urn, - aspect_type=GlobalTagsClass, - aspect="globalTags", - ) - assert len(tags_read.tags) == 1 - assert tags_read.tags[0].tag == new_tag.tag - - -def test_dataset_terms_patch(wait_for_healthchecks): - def get_terms(graph, dataset_urn): - return graph.get_aspect_v2( - entity_urn=dataset_urn, - aspect_type=GlossaryTermsClass, - aspect="glossaryTerms", - ) - - dataset_urn = make_dataset_urn( - platform="hive", name=f"SampleHiveDataset-{uuid.uuid4()}", env="PROD" - ) - - term_urn = make_term_urn(term=f"testTerm-{uuid.uuid4()}") - - term_association = GlossaryTermAssociationClass(urn=term_urn, context="test") - global_terms = GlossaryTermsClass( - terms=[term_association], - auditStamp=AuditStampClass( - time=int(time.time() * 1000.0), actor=make_user_urn("tester") - ), - ) - mcpw = MetadataChangeProposalWrapper(entityUrn=dataset_urn, aspect=global_terms) - - with DataHubGraph(DataHubGraphConfig()) as graph: - graph.emit_mcp(mcpw) - terms_read = get_terms(graph, dataset_urn) - assert terms_read.terms[0].urn == term_urn - assert terms_read.terms[0].context == "test" - - new_term = GlossaryTermAssociationClass( - urn=make_term_urn(f"test-{uuid.uuid4()}") - ) - for patch_mcp in DatasetPatchBuilder(dataset_urn).add_term(new_term).build(): - graph.emit_mcp(patch_mcp) - pass - - terms_read = get_terms(graph, dataset_urn) - - assert terms_read.terms[0].urn == term_urn - assert terms_read.terms[0].context == "test" - assert terms_read.terms[1].urn == new_term.urn - assert terms_read.terms[1].context is None - - for patch_mcp in DatasetPatchBuilder(dataset_urn).remove_term(term_urn).build(): - graph.emit_mcp(patch_mcp) - pass - - terms_read = get_terms(graph, dataset_urn) - assert len(terms_read.terms) == 1 - assert terms_read.terms[0].urn == new_term.urn - - def get_field_info( graph: DataHubGraph, dataset_urn: str, field_path: str ) -> Optional[EditableSchemaFieldInfoClass]: @@ -403,92 +284,10 @@ def test_custom_properties_patch(wait_for_healthchecks): dataset_urn = make_dataset_urn( platform="hive", name=f"SampleHiveDataset-{uuid.uuid4()}", env="PROD" ) - - base_property_map = {"base_property": "base_property_value"} - - orig_dataset_properties = DatasetPropertiesClass( - customProperties=base_property_map, - name="test dataset", - description="base description", - ) - mcpw = MetadataChangeProposalWrapper( - entityUrn=dataset_urn, aspect=orig_dataset_properties - ) + orig_dataset_properties = DatasetPropertiesClass(name="test_name", description="test_description") + helper_test_custom_properties_patch(test_entity_urn=dataset_urn, patch_builder_class=DatasetPatchBuilder, custom_properties_aspect_class=DatasetPropertiesClass, base_aspect=orig_dataset_properties) with DataHubGraph(DataHubGraphConfig()) as graph: - graph.emit(mcpw) - # assert custom properties looks as expected - custom_properties = get_custom_properties(graph, dataset_urn) - assert custom_properties - for k, v in base_property_map.items(): - assert custom_properties[k] == v - - new_properties = { - "test_property": "test_value", - "test_property1": "test_value1", - } - - dataset_patch_builder = DatasetPatchBuilder( - dataset_urn - ) - for k, v in new_properties.items(): - dataset_patch_builder.add_custom_property(k, v) - - for patch_mcp in dataset_patch_builder.build(): - graph.emit_mcp(patch_mcp) - - custom_properties = get_custom_properties(graph, dataset_urn) - - assert custom_properties is not None - for k, v in new_properties.items(): - assert custom_properties[k] == v - - # ensure exising properties were not touched - for k, v in base_property_map.items(): - assert custom_properties[k] == v - - # Remove property - for patch_mcp in ( - DatasetPatchBuilder(dataset_urn) - .remove_custom_property("test_property") - .build() - ): - graph.emit_mcp(patch_mcp) - - custom_properties = get_custom_properties(graph, dataset_urn) - - assert custom_properties is not None - assert "test_property" not in custom_properties - assert custom_properties["test_property1"] == "test_value1" - - # ensure exising properties were not touched - for k, v in base_property_map.items(): - assert custom_properties[k] == v - - # Replace custom properties - for patch_mcp in ( - DatasetPatchBuilder(dataset_urn) - .set_custom_properties(new_properties) - .build() - ): - graph.emit_mcp(patch_mcp) - - custom_properties = get_custom_properties(graph, dataset_urn) - - assert custom_properties is not None - for k in base_property_map: - assert k not in custom_properties - for k, v in new_properties.items(): - assert custom_properties[k] == v - - dataset_properties: Optional[DatasetPropertiesClass] = graph.get_aspect( - dataset_urn, DatasetPropertiesClass - ) - - assert dataset_properties - assert dataset_properties.name == orig_dataset_properties.name - assert dataset_properties.description == orig_dataset_properties.description - # Patch custom properties along with name for patch_mcp in ( DatasetPatchBuilder(dataset_urn) @@ -509,8 +308,6 @@ def test_custom_properties_patch(wait_for_healthchecks): custom_properties = get_custom_properties(graph, dataset_urn) assert custom_properties is not None - for k, v in new_properties.items(): - assert custom_properties[k] == v assert ( custom_properties["test_description_property"] == "test_description_value" From 66806a805ebe606257d66fa107ba95c3976cd867 Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Thu, 15 Jun 2023 03:05:49 -0400 Subject: [PATCH 017/222] feat(ingest/unity): Set external url for containers and datasets (#8238) --- .../ingestion/source/common/subtypes.py | 2 +- .../ingestion/source/sql/presto_on_hive.py | 2 +- .../datahub/ingestion/source/unity/config.py | 28 +- .../ingestion/source/unity/connection_test.py | 5 +- .../ingestion/source/unity/profiler.py | 1 - .../datahub/ingestion/source/unity/proxy.py | 271 ++++++++---------- .../ingestion/source/unity/proxy_types.py | 65 +++-- .../datahub/ingestion/source/unity/report.py | 3 +- .../datahub/ingestion/source/unity/source.py | 55 ++-- .../datahub/ingestion/source/unity/usage.py | 6 +- .../unity/test_unity_catalog_ingest.py | 111 +++---- .../unity/unity_catalog_mces_golden.json | 22 ++ .../tests/unit/test_unity_catalog_config.py | 30 -- 13 files changed, 249 insertions(+), 352 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py index 5154125c7ac9e..a2d89d26112f4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py @@ -22,7 +22,7 @@ class DatasetContainerSubTypes(str, Enum): DATABASE = "Database" SCHEMA = "Schema" # System-Specific SubTypes - PRESTO_CATALOG = "Catalog" + CATALOG = "Catalog" # Presto or Unity Catalog BIGQUERY_PROJECT = "Project" BIGQUERY_DATASET = "Dataset" DATABRICKS_METASTORE = "Metastore" diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py b/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py index b0cffc52d7cd2..2afab226a8461 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py @@ -297,7 +297,7 @@ def __init__(self, config: PrestoOnHiveConfig, ctx: PipelineContext) -> None: self.config: PrestoOnHiveConfig = config self._alchemy_client = SQLAlchemyClient(config) self.database_container_subtype = ( - DatasetContainerSubTypes.PRESTO_CATALOG + DatasetContainerSubTypes.CATALOG if config.use_catalog_subtype else DatasetContainerSubTypes.DATABASE ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/config.py b/metadata-ingestion/src/datahub/ingestion/source/unity/config.py index 29fb577633bc9..9146453f9c5be 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/config.py @@ -7,6 +7,7 @@ from datahub.configuration.common import AllowDenyPattern, ConfigModel from datahub.configuration.source_common import DatasetSourceConfigMixin +from datahub.configuration.validate_field_removal import pydantic_removed_field from datahub.configuration.validate_field_rename import pydantic_renamed_field from datahub.ingestion.source.state.stale_entity_removal_handler import ( StatefulStaleMetadataRemovalConfig, @@ -88,19 +89,11 @@ class UnityCatalogSourceConfig( description="Name of the workspace. Default to deployment name present in workspace_url", ) - only_ingest_assigned_metastore: bool = pydantic.Field( - default=False, - description=( - "Only ingest the workspace's currently assigned metastore. " - "Use if you only want to ingest one metastore and " - "do not want to grant your ingestion service account the admin role." - ), + _only_ingest_assigned_metastore_removed = pydantic_removed_field( + "only_ingest_assigned_metastore" ) - metastore_id_pattern: AllowDenyPattern = Field( - default=AllowDenyPattern.allow_all(), - description="Regex patterns for metastore id to filter in ingestion.", - ) + _metastore_id_pattern_removed = pydantic_removed_field("metastore_id_pattern") catalog_pattern: AllowDenyPattern = Field( default=AllowDenyPattern.allow_all(), @@ -153,19 +146,6 @@ class UnityCatalogSourceConfig( default=None, description="Unity Catalog Stateful Ingestion Config." ) - @pydantic.validator("metastore_id_pattern") - def no_metastore_pattern_if_only_ingest_assigned_metastore( - cls, v: bool, values: Dict[str, Any] - ) -> bool: - if ( - values.get("only_ingest_assigned_metastore") - and v != AllowDenyPattern.allow_all() - ): - raise ValueError( - "metastore_id_pattern cannot be set when only_ingest_assigned_metastore is specified." - ) - return v - @pydantic.validator("start_time") def within_thirty_days(cls, v: datetime) -> datetime: if (datetime.now(timezone.utc) - v).days > 30: diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/connection_test.py b/metadata-ingestion/src/datahub/ingestion/source/unity/connection_test.py index 1d16c3e3574fa..c7be2badc4078 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/connection_test.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/connection_test.py @@ -35,10 +35,7 @@ def get_connection_test(self) -> TestConnectionReport: def basic_connectivity(self) -> CapabilityReport: try: - if self.config.only_ingest_assigned_metastore: - return CapabilityReport(capable=self.proxy.check_basic_connectivity()) - else: - return CapabilityReport(capable=self.proxy.check_account_admin()) + return CapabilityReport(capable=self.proxy.check_basic_connectivity()) except Exception as e: return CapabilityReport(capable=False, failure_reason=str(e)) diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/profiler.py b/metadata-ingestion/src/datahub/ingestion/source/unity/profiler.py index 9609f419973e4..8066932e3afe9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/profiler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/profiler.py @@ -39,7 +39,6 @@ def get_workunits( for future in as_completed(futures): wu: Optional[MetadataWorkUnit] = future.result() if wu: - self.report.num_profile_workunits_emitted += 1 yield wu except Exception as e: self.report.report_warning("profiling", str(e)) diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py index 48a95cbf4b678..4828f2171fb9c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py @@ -5,25 +5,31 @@ import logging from datetime import datetime from typing import Any, Dict, Iterable, List, Optional, Union +from unittest.mock import patch from databricks.sdk import WorkspaceClient -from databricks.sdk.service.catalog import GetMetastoreSummaryResponse, MetastoreInfo +from databricks.sdk.service.catalog import ( + CatalogInfo, + ColumnInfo, + GetMetastoreSummaryResponse, + MetastoreInfo, + SchemaInfo, + TableInfo, +) +from databricks.sdk.service.iam import ServicePrincipal as DatabricksServicePrincipal from databricks.sdk.service.sql import ( QueryFilter, QueryInfo, QueryStatementType, QueryStatus, ) -from databricks_cli.sdk.api_client import ApiClient -from databricks_cli.unity_catalog.api import UnityCatalogApi -from datahub.ingestion.source.common.subtypes import DatasetContainerSubTypes +import datahub from datahub.ingestion.source.unity.proxy_profiling import ( UnityCatalogProxyProfilingMixin, ) from datahub.ingestion.source.unity.proxy_types import ( ALLOWED_STATEMENT_TYPES, - DATA_TYPE_REGISTRY, Catalog, Column, Metastore, @@ -34,11 +40,24 @@ TableReference, ) from datahub.ingestion.source.unity.report import UnityCatalogReport -from datahub.metadata.schema_classes import SchemaFieldDataTypeClass logger: logging.Logger = logging.getLogger(__name__) +class TableInfoWithGeneration(TableInfo): + generation: Optional[int] = None + + @classmethod + def as_dict(self) -> dict: + return {**super().as_dict(), "generation": self.generation} + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> "TableInfoWithGeneration": + table_info = super().from_dict(d) + table_info.generation = d.get("generation") + return table_info + + @dataclasses.dataclass class QueryFilterWithStatementTypes(QueryFilter): statement_types: List[QueryStatementType] = dataclasses.field(default_factory=list) @@ -55,7 +74,6 @@ def from_dict(cls, d: Dict[str, Any]) -> "QueryFilterWithStatementTypes": class UnityCatalogApiProxy(UnityCatalogProxyProfilingMixin): _workspace_client: WorkspaceClient - _unity_catalog_api: UnityCatalogApi _workspace_url: str report: UnityCatalogReport warehouse_id: str @@ -68,18 +86,14 @@ def __init__( report: UnityCatalogReport, ): self._workspace_client = WorkspaceClient( - host=workspace_url, token=personal_access_token - ) - self._unity_catalog_api = UnityCatalogApi( - ApiClient(host=workspace_url, token=personal_access_token) + host=workspace_url, + token=personal_access_token, + product="datahub", + product_version=datahub.nice_version_name(), ) self.warehouse_id = warehouse_id or "" self.report = report - def check_account_admin(self) -> bool: - self._workspace_client.metastores.list() - return True - def check_basic_connectivity(self) -> bool: self._workspace_client.metastores.summary() return True @@ -88,72 +102,40 @@ def assigned_metastore(self) -> Metastore: response = self._workspace_client.metastores.summary() return self._create_metastore(response) - def metastores(self) -> Iterable[Metastore]: - response = self._workspace_client.metastores.list() - if not response: - logger.info("Metastores not found") - return [] - for metastore in response: - yield self._create_metastore(metastore) - def catalogs(self, metastore: Metastore) -> Iterable[Catalog]: - response: dict = self._unity_catalog_api.list_catalogs() - num_catalogs: int = 0 - if response.get("catalogs") is None: - logger.info(f"Catalogs not found for metastore {metastore.name}") + response = self._workspace_client.catalogs.list() + if not response: + logger.info("Catalogs not found") return [] - - for obj in response["catalogs"]: - if obj["metastore_id"] == metastore.metastore_id: - yield self._create_catalog(metastore, obj) - num_catalogs += 1 - - if num_catalogs == 0: - logger.info( - f"Catalogs not found for metastore where metastore_id is {metastore.metastore_id}" - ) + for catalog in response: + yield self._create_catalog(metastore, catalog) def schemas(self, catalog: Catalog) -> Iterable[Schema]: - response: dict = self._unity_catalog_api.list_schemas( - catalog_name=catalog.name, name_pattern=None - ) - if response.get("schemas") is None: - logger.info(f"Schemas not found for catalog {catalog.name}") + response = self._workspace_client.schemas.list(catalog_name=catalog.name) + if not response: + logger.info(f"Schemas not found for catalog {catalog.id}") return [] - - for schema in response["schemas"]: + for schema in response: yield self._create_schema(catalog, schema) def tables(self, schema: Schema) -> Iterable[Table]: - response: dict = self._unity_catalog_api.list_tables( - catalog_name=schema.catalog.name, - schema_name=schema.name, - name_pattern=None, - ) - - if response.get("tables") is None: - logger.info( - f"Tables not found for schema {schema.catalog.name}.{schema.name}" + with patch("databricks.sdk.service.catalog.TableInfo", TableInfoWithGeneration): + response = self._workspace_client.tables.list( + catalog_name=schema.catalog.name, schema_name=schema.name ) - return [] - - for table in response["tables"]: - yield self._create_table(schema=schema, obj=table) + if not response: + logger.info(f"Tables not found for schema {schema.id}") + return [] + for table in response: + try: + yield self._create_table(schema, table) + except Exception as e: + logger.warning(f"Error parsing table: {e}") + self.report.report_warning("table-parse", str(e)) def service_principals(self) -> Iterable[ServicePrincipal]: - # TODO: Replace with self._workspace_client.service_principals.list() when it supports pagination - start_index = 1 # Unfortunately 1-indexed - items_per_page = 0 - total_results = float("inf") - while start_index + items_per_page <= total_results: - response: dict = self._unity_catalog_api.client.client.perform_query( - "GET", "/account/scim/v2/ServicePrincipals" - ) - start_index = response["startIndex"] - items_per_page = response["itemsPerPage"] - total_results = response["totalResults"] - for principal in response["Resources"]: - yield self._create_service_principal(principal) + for principal in self._workspace_client.service_principals.list(): + yield self._create_service_principal(principal) def query_history( self, @@ -214,39 +196,20 @@ def _query_history( method, path, body={**body, "page_token": response["next_page_token"]} ) - def list_lineages_by_table(self, table_name=None, headers=None): - """ - List table lineage by table name - """ - _data = {} - if table_name is not None: - _data["table_name"] = table_name - - return self._unity_catalog_api.client.client.perform_query( - "GET", - "/lineage-tracking/table-lineage/get", - data=_data, - headers=headers, - version="2.0", + def list_lineages_by_table(self, table_name: str) -> dict: + """List table lineage by table name.""" + return self._workspace_client.api_client.do( + method="GET", + path="/api/2.0/lineage-tracking/table-lineage/get", + body={"table_name": table_name}, ) - def list_lineages_by_column(self, table_name=None, column_name=None, headers=None): - """ - List column lineage by table name and comlumn name - """ - # Lineage endpoint doesn't exists on 2.1 version - _data = {} - if table_name is not None: - _data["table_name"] = table_name - if column_name is not None: - _data["column_name"] = column_name - - return self._unity_catalog_api.client.client.perform_query( + def list_lineages_by_column(self, table_name: str, column_name: str) -> dict: + """List column lineage by table name and column name.""" + return self._workspace_client.api_client.do( "GET", - "/lineage-tracking/column-lineage/get", - data=_data, - headers=headers, - version="2.0", + "/api/2.0/lineage-tracking/column-lineage/get", + body={"table_name": table_name, "column_name": column_name}, ) def table_lineage(self, table: Table) -> None: @@ -303,90 +266,80 @@ def _create_metastore( return Metastore( name=obj.name, id=UnityCatalogApiProxy._escape_sequence(obj.name), + global_metastore_id=obj.global_metastore_id, metastore_id=obj.metastore_id, - type=DatasetContainerSubTypes.DATABRICKS_METASTORE, owner=obj.owner, region=obj.region, cloud=obj.cloud, comment=None, ) - def _create_catalog(self, metastore: Metastore, obj: Any) -> Catalog: + def _create_catalog(self, metastore: Metastore, obj: CatalogInfo) -> Catalog: return Catalog( - name=obj["name"], - id="{}.{}".format( - metastore.id, - self._escape_sequence(obj["name"]), - ), + name=obj.name, + id="{}.{}".format(metastore.id, self._escape_sequence(obj.name)), metastore=metastore, - type="Catalog", - comment=obj.get("comment"), - owner=obj.get("owner"), + comment=obj.comment, + owner=obj.owner, + type=obj.catalog_type, ) - def _create_schema(self, catalog: Catalog, obj: Any) -> Schema: + def _create_schema(self, catalog: Catalog, obj: SchemaInfo) -> Schema: return Schema( - name=obj["name"], - id="{}.{}".format( - catalog.id, - self._escape_sequence(obj["name"]), - ), + name=obj.name, + id="{}.{}".format(catalog.id, self._escape_sequence(obj.name)), catalog=catalog, - type="Schema", - comment=obj.get("comment"), - owner=obj.get("owner"), + comment=obj.comment, + owner=obj.owner, ) - def _create_column(self, table_id: str, obj: Any) -> Column: + def _create_column(self, table_id: str, obj: ColumnInfo) -> Column: return Column( - name=obj["name"], - id="{}.{}".format(table_id, self._escape_sequence(obj["name"])), - type_text=obj["type_text"], - type_name=SchemaFieldDataTypeClass( - type=DATA_TYPE_REGISTRY[obj["type_name"]]() - ), - type_scale=obj["type_scale"], - type_precision=obj["type_precision"], - position=obj["position"], - nullable=obj["nullable"], - comment=obj.get("comment"), - type="Column", + name=obj.name, + id="{}.{}".format(table_id, self._escape_sequence(obj.name)), + type_text=obj.type_text, + type_name=obj.type_name, + type_scale=obj.type_scale, + type_precision=obj.type_precision, + position=obj.position, + nullable=obj.nullable, + comment=obj.comment, ) - def _create_table(self, schema: Schema, obj: Any) -> Table: - table_id: str = "{}.{}".format(schema.id, self._escape_sequence(obj["name"])) + def _create_table(self, schema: Schema, obj: TableInfoWithGeneration) -> Table: + table_id: str = "{}.{}".format(schema.id, self._escape_sequence(obj.name)) return Table( - name=obj["name"], + name=obj.name, id=table_id, - table_type=obj["table_type"], + table_type=obj.table_type, schema=schema, - storage_location=obj.get("storage_location"), - data_source_format=obj.get("data_source_format"), - columns=[self._create_column(table_id, column) for column in obj["columns"]] - if obj.get("columns") is not None - else [], - type="view" if str(obj["table_type"]).lower() == "view" else "table", - view_definition=obj.get("view_definition", None), - properties=obj.get("properties", {}), - owner=obj.get("owner"), - generation=obj["generation"], - created_at=datetime.utcfromtimestamp(obj["created_at"] / 1000), - created_by=obj["created_by"], - updated_at=datetime.utcfromtimestamp(obj["updated_at"] / 1000) - if "updated_at" in obj + storage_location=obj.storage_location, + data_source_format=obj.data_source_format, + columns=[ + self._create_column(table_id, column) for column in obj.columns or [] + ], + view_definition=obj.view_definition or None, + properties=obj.properties or {}, + owner=obj.owner, + generation=obj.generation, + created_at=datetime.utcfromtimestamp(obj.created_at / 1000), + created_by=obj.created_by, + updated_at=datetime.utcfromtimestamp(obj.updated_at / 1000) + if obj.updated_at else None, - updated_by=obj.get("updated_by", None), - table_id=obj["table_id"], - comment=obj.get("comment"), + updated_by=obj.updated_by, + table_id=obj.table_id, + comment=obj.comment, ) - def _create_service_principal(self, obj: dict) -> ServicePrincipal: - display_name = obj["displayName"] + def _create_service_principal( + self, obj: DatabricksServicePrincipal + ) -> ServicePrincipal: return ServicePrincipal( - id="{}.{}".format(obj["id"], self._escape_sequence(display_name)), - display_name=display_name, - application_id=obj["applicationId"], - active=obj.get("active"), + id="{}.{}".format(obj.id, self._escape_sequence(obj.display_name)), + display_name=obj.display_name, + application_id=obj.application_id, + active=obj.active, ) @staticmethod diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py index cb32ff1290f55..2b943d8c98e7d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py @@ -4,6 +4,12 @@ from datetime import datetime from typing import Dict, List, Optional +from databricks.sdk.service.catalog import ( + CatalogType, + ColumnTypeName, + DataSourceFormat, + TableType, +) from databricks.sdk.service.sql import QueryStatementType from datahub.metadata.schema_classes import ( @@ -16,30 +22,30 @@ NumberTypeClass, OperationTypeClass, RecordTypeClass, - SchemaFieldDataTypeClass, StringTypeClass, TimeTypeClass, ) DATA_TYPE_REGISTRY: dict = { - "BOOLEAN": BooleanTypeClass, - "BYTE": BytesTypeClass, - "DATE": DateTypeClass, - "SHORT": NumberTypeClass, - "INT": NumberTypeClass, - "LONG": NumberTypeClass, - "FLOAT": NumberTypeClass, - "DOUBLE": NumberTypeClass, - "TIMESTAMP": TimeTypeClass, - "STRING": StringTypeClass, - "BINARY": BytesTypeClass, - "DECIMAL": NumberTypeClass, - "INTERVAL": TimeTypeClass, - "ARRAY": ArrayTypeClass, - "STRUCT": RecordTypeClass, - "MAP": MapTypeClass, - "CHAR": StringTypeClass, - "NULL": NullTypeClass, + ColumnTypeName.BOOLEAN: BooleanTypeClass, + ColumnTypeName.BYTE: BytesTypeClass, + ColumnTypeName.DATE: DateTypeClass, + ColumnTypeName.SHORT: NumberTypeClass, + ColumnTypeName.INT: NumberTypeClass, + ColumnTypeName.LONG: NumberTypeClass, + ColumnTypeName.FLOAT: NumberTypeClass, + ColumnTypeName.DOUBLE: NumberTypeClass, + ColumnTypeName.TIMESTAMP: TimeTypeClass, + ColumnTypeName.TIMESTAMP_NTZ: TimeTypeClass, + ColumnTypeName.STRING: StringTypeClass, + ColumnTypeName.BINARY: BytesTypeClass, + ColumnTypeName.DECIMAL: NumberTypeClass, + ColumnTypeName.INTERVAL: TimeTypeClass, + ColumnTypeName.ARRAY: ArrayTypeClass, + ColumnTypeName.STRUCT: RecordTypeClass, + ColumnTypeName.MAP: MapTypeClass, + ColumnTypeName.CHAR: StringTypeClass, + ColumnTypeName.NULL: NullTypeClass, } @@ -64,12 +70,12 @@ class CommonProperty: id: str name: str - type: str comment: Optional[str] @dataclass class Metastore(CommonProperty): + global_metastore_id: str # Global across clouds and regions metastore_id: str owner: Optional[str] cloud: Optional[str] @@ -80,6 +86,7 @@ class Metastore(CommonProperty): class Catalog(CommonProperty): metastore: Metastore owner: Optional[str] + type: CatalogType @dataclass @@ -91,7 +98,7 @@ class Schema(CommonProperty): @dataclass class Column(CommonProperty): type_text: str - type_name: SchemaFieldDataTypeClass + type_name: ColumnTypeName type_precision: int type_scale: int position: int @@ -115,7 +122,7 @@ class ServicePrincipal: @dataclass(frozen=True, order=True) class TableReference: - metastore_id: str + metastore: str catalog: str schema: str table: str @@ -130,23 +137,27 @@ def create(cls, table: "Table") -> "TableReference": ) def __str__(self) -> str: - return f"{self.metastore_id}.{self.catalog}.{self.schema}.{self.table}" + return f"{self.metastore}.{self.catalog}.{self.schema}.{self.table}" @property def qualified_table_name(self) -> str: return f"{self.catalog}.{self.schema}.{self.table}" + @property + def external_path(self) -> str: + return f"{self.catalog}/{self.schema}/{self.table}" + @dataclass class Table(CommonProperty): schema: Schema columns: List[Column] storage_location: Optional[str] - data_source_format: Optional[str] + data_source_format: Optional[DataSourceFormat] comment: Optional[str] - table_type: str + table_type: TableType owner: Optional[str] - generation: int + generation: Optional[int] created_at: datetime created_by: str updated_at: Optional[datetime] @@ -157,10 +168,10 @@ class Table(CommonProperty): upstreams: Dict[TableReference, Dict[str, List[str]]] = field(default_factory=dict) ref: TableReference = field(init=False) - # lineage: Optional[Lineage] def __post_init__(self): self.ref = TableReference.create(self) + self.is_view = self.table_type in [TableType.VIEW, TableType.MATERIALIZED_VIEW] @dataclass diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/report.py b/metadata-ingestion/src/datahub/ingestion/source/unity/report.py index 2b6fa649ebcaa..8382b31a56add 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/report.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/report.py @@ -22,8 +22,8 @@ class UnityCatalogReport(StaleEntityRemovalSourceReport): num_queries_dropped_duplicate_table: int = 0 num_queries_parsed_by_spark_plan: int = 0 + # Distinguish from Operations emitted for created / updated timestamps num_operational_stats_workunits_emitted: int = 0 - num_usage_workunits_emitted: int = 0 profile_table_timeouts: LossyList[str] = field(default_factory=LossyList) profile_table_empty: LossyList[str] = field(default_factory=LossyList) @@ -32,4 +32,3 @@ class UnityCatalogReport(StaleEntityRemovalSourceReport): ) num_profile_failed_unsupported_column_type: int = 0 num_profile_failed_int_casts: int = 0 - num_profile_workunits_emitted: int = 0 diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py index de22feaf6af33..5e152351b0a3e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py @@ -3,6 +3,7 @@ import time from datetime import timedelta from typing import Dict, Iterable, List, Optional, Set +from urllib.parse import urljoin from datahub.emitter.mce_builder import ( make_data_platform_urn, @@ -50,6 +51,7 @@ from datahub.ingestion.source.unity.profiler import UnityCatalogProfiler from datahub.ingestion.source.unity.proxy import UnityCatalogApiProxy from datahub.ingestion.source.unity.proxy_types import ( + DATA_TYPE_REGISTRY, Catalog, Column, Metastore, @@ -70,12 +72,14 @@ DatasetPropertiesClass, DomainsClass, MySqlDDLClass, + NullTypeClass, OperationClass, OperationTypeClass, OwnerClass, OwnershipClass, OwnershipTypeClass, SchemaFieldClass, + SchemaFieldDataTypeClass, SchemaMetadataClass, SubTypesClass, TimeStampClass, @@ -132,6 +136,7 @@ def __init__(self, ctx: PipelineContext, config: UnityCatalogSourceConfig): config.profiling.warehouse_id, report=self.report, ) + self.external_url_base = urljoin(self.config.workspace_url, "/explore/data") # Determine the platform_instance_name self.platform_instance_name = ( @@ -215,24 +220,11 @@ def build_service_principal_map(self) -> None: ) def process_metastores(self) -> Iterable[MetadataWorkUnit]: - metastores: Dict[str, Metastore] = {} - assigned_metastore = self.unity_catalog_api_proxy.assigned_metastore() - metastores[assigned_metastore.metastore_id] = assigned_metastore + metastore = self.unity_catalog_api_proxy.assigned_metastore() + yield from self.gen_metastore_containers(metastore) + yield from self.process_catalogs(metastore) - if not self.config.only_ingest_assigned_metastore: - for metastore in self.unity_catalog_api_proxy.metastores(): - metastores[metastore.metastore_id] = metastore - - for metastore in metastores.values(): - if not self.config.metastore_id_pattern.allowed(metastore.metastore_id): - self.report.metastores.dropped(metastore.metastore_id) - continue - - logger.info(f"Started to process metastore: {metastore.metastore_id}") - yield from self.gen_metastore_containers(metastore) - yield from self.process_catalogs(metastore) - - self.report.metastores.processed(metastore.metastore_id) + self.report.metastores.processed(metastore.id) def process_catalogs(self, metastore: Metastore) -> Iterable[MetadataWorkUnit]: for catalog in self.unity_catalog_api_proxy.catalogs(metastore=metastore): @@ -259,15 +251,15 @@ def process_schemas(self, catalog: Catalog) -> Iterable[MetadataWorkUnit]: def process_tables(self, schema: Schema) -> Iterable[MetadataWorkUnit]: for table in self.unity_catalog_api_proxy.tables(schema=schema): if not self.config.table_pattern.allowed(table.ref.qualified_table_name): - self.report.tables.dropped(table.id, type=table.type) + self.report.tables.dropped(table.id, f"table ({table.table_type})") continue - if table.type.lower() == "view": + if table.is_view: self.view_refs.add(table.ref) else: self.table_refs.add(table.ref) yield from self.process_table(table, schema) - self.report.tables.processed(table.id, type=table.type) + self.report.tables.processed(table.id, f"table ({table.table_type})") def process_table(self, table: Table, schema: Schema) -> Iterable[MetadataWorkUnit]: dataset_urn = self.gen_dataset_urn(table.ref) @@ -403,6 +395,7 @@ def gen_schema_containers(self, schema: Schema) -> Iterable[MetadataWorkUnit]: domain_urn=domain_urn, description=schema.comment, owner_urn=self.get_owner_urn(schema.owner), + external_url=f"{self.external_url_base}/{schema.catalog.name}/{schema.name}", ) def gen_metastore_containers( @@ -418,6 +411,7 @@ def gen_metastore_containers( domain_urn=domain_urn, description=metastore.comment, owner_urn=self.get_owner_urn(metastore.owner), + external_url=self.external_url_base, ) def gen_catalog_containers(self, catalog: Catalog) -> Iterable[MetadataWorkUnit]: @@ -428,11 +422,12 @@ def gen_catalog_containers(self, catalog: Catalog) -> Iterable[MetadataWorkUnit] yield from gen_containers( container_key=catalog_container_key, name=catalog.name, - sub_types=[DatasetContainerSubTypes.PRESTO_CATALOG], + sub_types=[DatasetContainerSubTypes.CATALOG], domain_urn=domain_urn, parent_container_key=metastore_container_key, description=catalog.comment, owner_urn=self.get_owner_urn(catalog.owner), + external_url=f"{self.external_url_base}/{catalog.name}", ) def gen_schema_key(self, schema: Schema) -> PlatformKey: @@ -484,10 +479,11 @@ def _create_table_property_aspect(self, table: Table) -> DatasetPropertiesClass: if table.storage_location is not None: custom_properties["storage_location"] = table.storage_location if table.data_source_format is not None: - custom_properties["data_source_format"] = table.data_source_format + custom_properties["data_source_format"] = table.data_source_format.value + if table.generation is not None: + custom_properties["generation"] = str(table.generation) - custom_properties["generation"] = str(table.generation) - custom_properties["table_type"] = table.table_type + custom_properties["table_type"] = table.table_type.value custom_properties["created_by"] = table.created_by custom_properties["created_at"] = str(table.created_at) @@ -515,6 +511,7 @@ def _create_table_property_aspect(self, table: Table) -> DatasetPropertiesClass: customProperties=custom_properties, created=created, lastModified=last_modified, + externalUrl=f"{self.external_url_base}/{table.ref.external_path}", ) def _create_table_operation_aspect(self, table: Table) -> OperationClass: @@ -561,11 +558,7 @@ def _create_table_ownership_aspect(self, table: Table) -> Optional[OwnershipClas def _create_table_sub_type_aspect(self, table: Table) -> SubTypesClass: return SubTypesClass( - typeNames=[ - DatasetSubTypes.VIEW - if table.table_type.lower() == "view" - else DatasetSubTypes.TABLE - ] + typeNames=[DatasetSubTypes.VIEW if table.is_view else DatasetSubTypes.TABLE] ) def _create_view_property_aspect(self, table: Table) -> ViewProperties: @@ -601,7 +594,9 @@ def _create_schema_field(column: Column) -> List[SchemaFieldClass]: return [ SchemaFieldClass( fieldPath=column.name, - type=column.type_name, + type=SchemaFieldDataTypeClass( + type=DATA_TYPE_REGISTRY.get(column.type_name, NullTypeClass)() + ), nativeDataType=column.type_text, nullable=column.nullable, description=column.comment, diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/usage.py b/metadata-ingestion/src/datahub/ingestion/source/unity/usage.py index 7bb09b10526bc..befa1fdaebecc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/usage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/usage.py @@ -100,12 +100,10 @@ def _generate_workunits(self, table_map: TableMap) -> Iterable[MetadataWorkUnit] ) return - for wu in self.usage_aggregator.generate_workunits( + yield from self.usage_aggregator.generate_workunits( resource_urn_builder=self.table_urn_builder, user_urn_builder=self.user_urn_builder, - ): - self.report.num_usage_workunits_emitted += 1 - yield wu + ) def _generate_operation_workunit( self, query: Query, table_info: QueryTableInfo diff --git a/metadata-ingestion/tests/integration/unity/test_unity_catalog_ingest.py b/metadata-ingestion/tests/integration/unity/test_unity_catalog_ingest.py index 80bc36df4b4f9..c43ba7eee5847 100644 --- a/metadata-ingestion/tests/integration/unity/test_unity_catalog_ingest.py +++ b/metadata-ingestion/tests/integration/unity/test_unity_catalog_ingest.py @@ -1,7 +1,14 @@ import uuid from unittest import mock +from unittest.mock import patch -from databricks.sdk.service.catalog import GetMetastoreSummaryResponse +import databricks +from databricks.sdk.service.catalog import ( + CatalogInfo, + GetMetastoreSummaryResponse, + SchemaInfo, +) +from databricks.sdk.service.iam import ServicePrincipal from freezegun import freeze_time from datahub.ingestion.run.pipeline import Pipeline @@ -30,7 +37,7 @@ def register_mock_api(request_mock): ) -def register_mock_data(unity_catalog_api_instance, workspace_client): +def register_mock_data(workspace_client): workspace_client.metastores.summary.return_value = GetMetastoreSummaryResponse.from_dict( { "name": "acryl metastore", @@ -52,26 +59,9 @@ def register_mock_data(unity_catalog_api_instance, workspace_client): } ) - unity_catalog_api_instance.get_metastore_summary.return_value = { - "name": "acryl metastore", - "storage_root": "s3://db-9063-5b7e3b6d3736-s3-root-bucket/metastore/2c983545-d403-4f87-9063-5b7e3b6d3736", - "default_data_access_config_id": "9a9bacc4-cd82-409f-b55c-8ad1b2bde8da", - "storage_root_credential_id": "9a9bacc4-cd82-409f-b55c-8ad1b2bde8da", - "storage_root_credential_name": "2c983545-d403-4f87-9063-5b7e3b6d3736-data-access-config-1666185153576", - "delta_sharing_scope": "INTERNAL", - "owner": "abc@acryl.io", - "privilege_model_version": "1.0", - "region": "us-west-1", - "metastore_id": "2c983545-d403-4f87-9063-5b7e3b6d3736", - "created_at": 1666185153375, - "created_by": "abc@acryl.io", - "updated_at": 1666185154797, - "updated_by": "abc@acryl.io", - "cloud": "aws", - "global_metastore_id": "aws:us-west-1:2c983545-d403-4f87-9063-5b7e3b6d3736", - } - unity_catalog_api_instance.list_catalogs.return_value = { - "catalogs": [ + workspace_client.catalogs.list.return_value = [ + CatalogInfo.from_dict(d) + for d in [ { "name": "main", "owner": "account users", @@ -106,10 +96,11 @@ def register_mock_data(unity_catalog_api_instance, workspace_client): "catalog_type": "SYSTEM_CATALOG", }, ] - } + ] - unity_catalog_api_instance.list_schemas.return_value = { - "schemas": [ + workspace_client.schemas.list.return_value = [ + SchemaInfo.from_dict(d) + for d in [ { "name": "default", "catalog_name": "quickstart_catalog", @@ -151,9 +142,11 @@ def register_mock_data(unity_catalog_api_instance, workspace_client): "catalog_type": "MANAGED_CATALOG", }, ] - } - unity_catalog_api_instance.list_tables.return_value = { - "tables": [ + ] + + # Set as function so TableInfo can be patched + workspace_client.tables.list = lambda *args, **kwargs: [ + databricks.sdk.service.catalog.TableInfo.from_dict( { "name": "quickstart_table", "catalog_name": "quickstart_catalog", @@ -200,36 +193,26 @@ def register_mock_data(unity_catalog_api_instance, workspace_client): "updated_by": "abc@acryl.io", "table_id": "cff27aa1-1c6a-4d78-b713-562c660c2896", } - ] - } - + ) + ] -def mock_perform_query(method, path, *args, **kwargs): - if method == "GET" and path == "/account/scim/v2/ServicePrincipals": - return { - "Resources": [ - { - "displayName": "Service Principal 1", - "active": True, - "id": "4940620261358622", - "applicationId": SERVICE_PRINCIPAL_ID_1, - }, - { - "displayName": "Service Principal 2", - "active": True, - "id": "8202844266343024", - "applicationId": SERVICE_PRINCIPAL_ID_2, - }, - ], - "totalResults": 2, - "startIndex": 1, - "itemsPerPage": 2, - "schemas": ["urn:ietf:params:scim:api:messages:2.0:ListResponse"], - } - elif method == "get" and path == "/sql/history/queries": - return {"next_page_token": "next_token", "has_next_page": False, "res": []} - else: - return {} + workspace_client.service_principals.list.return_value = [ + ServicePrincipal.from_dict(d) + for d in [ + { + "displayName": "Service Principal 1", + "active": True, + "id": "4940620261358622", + "applicationId": SERVICE_PRINCIPAL_ID_1, + }, + { + "displayName": "Service Principal 2", + "active": True, + "id": "8202844266343024", + "applicationId": SERVICE_PRINCIPAL_ID_2, + }, + ] + ] @freeze_time(FROZEN_TIME) @@ -240,20 +223,10 @@ def test_ingestion(pytestconfig, tmp_path, requests_mock): output_file_name = "unity_catalog_mcps.json" - with mock.patch( - "databricks_cli.unity_catalog.api.UnityCatalogApi" - ) as UnityCatalogApi, mock.patch( - "databricks.sdk.WorkspaceClient" - ) as WorkspaceClient: - unity_catalog_api_instance: mock.MagicMock = mock.MagicMock() - unity_catalog_api_instance.client.client.perform_query.side_effect = ( - mock_perform_query - ) - UnityCatalogApi.return_value = unity_catalog_api_instance - + with patch("databricks.sdk.WorkspaceClient") as WorkspaceClient: workspace_client: mock.MagicMock = mock.MagicMock() WorkspaceClient.return_value = workspace_client - register_mock_data(unity_catalog_api_instance, workspace_client) + register_mock_data(workspace_client) config_dict: dict = { "run_id": "unity-catalog-test", diff --git a/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json b/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json index 59e4d755d220f..1b84d3132f281 100644 --- a/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json +++ b/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json @@ -10,6 +10,7 @@ "platform": "databricks", "metastore": "acryl metastore" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data", "name": "acryl metastore" } }, @@ -101,6 +102,7 @@ "metastore": "acryl metastore", "catalog": "main" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data/main", "name": "main", "description": "Main catalog (auto-created)" } @@ -209,6 +211,7 @@ "catalog": "main", "unity_schema": "default" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data/main/default", "name": "default", "description": "Default schema (auto-created)" } @@ -342,6 +345,7 @@ "updated_by": "abc@acryl.io", "updated_at": "2022-10-19 13:27:29.633000" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data/main/default/quickstart_table", "name": "quickstart_table", "qualifiedName": "main.default.quickstart_table", "created": { @@ -493,6 +497,7 @@ "catalog": "main", "unity_schema": "information_schema" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data/main/information_schema", "name": "information_schema", "description": "Information schema (auto-created)" } @@ -626,6 +631,7 @@ "updated_by": "abc@acryl.io", "updated_at": "2022-10-19 13:27:29.633000" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data/main/information_schema/quickstart_table", "name": "quickstart_table", "qualifiedName": "main.information_schema.quickstart_table", "created": { @@ -777,6 +783,7 @@ "catalog": "main", "unity_schema": "quickstart_schema" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data/main/quickstart_schema", "name": "quickstart_schema", "description": "A new Unity Catalog schema called quickstart_schema" } @@ -910,6 +917,7 @@ "updated_by": "abc@acryl.io", "updated_at": "2022-10-19 13:27:29.633000" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data/main/quickstart_schema/quickstart_table", "name": "quickstart_table", "qualifiedName": "main.quickstart_schema.quickstart_table", "created": { @@ -1060,6 +1068,7 @@ "metastore": "acryl metastore", "catalog": "quickstart_catalog" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data/quickstart_catalog", "name": "quickstart_catalog", "description": "" } @@ -1168,6 +1177,7 @@ "catalog": "quickstart_catalog", "unity_schema": "default" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data/quickstart_catalog/default", "name": "default", "description": "Default schema (auto-created)" } @@ -1301,6 +1311,7 @@ "updated_by": "abc@acryl.io", "updated_at": "2022-10-19 13:27:29.633000" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data/quickstart_catalog/default/quickstart_table", "name": "quickstart_table", "qualifiedName": "quickstart_catalog.default.quickstart_table", "created": { @@ -1452,6 +1463,7 @@ "catalog": "quickstart_catalog", "unity_schema": "information_schema" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data/quickstart_catalog/information_schema", "name": "information_schema", "description": "Information schema (auto-created)" } @@ -1585,6 +1597,7 @@ "updated_by": "abc@acryl.io", "updated_at": "2022-10-19 13:27:29.633000" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data/quickstart_catalog/information_schema/quickstart_table", "name": "quickstart_table", "qualifiedName": "quickstart_catalog.information_schema.quickstart_table", "created": { @@ -1736,6 +1749,7 @@ "catalog": "quickstart_catalog", "unity_schema": "quickstart_schema" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data/quickstart_catalog/quickstart_schema", "name": "quickstart_schema", "description": "A new Unity Catalog schema called quickstart_schema" } @@ -1869,6 +1883,7 @@ "updated_by": "abc@acryl.io", "updated_at": "2022-10-19 13:27:29.633000" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data/quickstart_catalog/quickstart_schema/quickstart_table", "name": "quickstart_table", "qualifiedName": "quickstart_catalog.quickstart_schema.quickstart_table", "created": { @@ -2019,6 +2034,7 @@ "metastore": "acryl metastore", "catalog": "system" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data/system", "name": "system", "description": "System catalog (auto-created)" } @@ -2127,6 +2143,7 @@ "catalog": "system", "unity_schema": "default" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data/system/default", "name": "default", "description": "Default schema (auto-created)" } @@ -2260,6 +2277,7 @@ "updated_by": "abc@acryl.io", "updated_at": "2022-10-19 13:27:29.633000" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data/system/default/quickstart_table", "name": "quickstart_table", "qualifiedName": "system.default.quickstart_table", "created": { @@ -2411,6 +2429,7 @@ "catalog": "system", "unity_schema": "information_schema" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data/system/information_schema", "name": "information_schema", "description": "Information schema (auto-created)" } @@ -2544,6 +2563,7 @@ "updated_by": "abc@acryl.io", "updated_at": "2022-10-19 13:27:29.633000" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data/system/information_schema/quickstart_table", "name": "quickstart_table", "qualifiedName": "system.information_schema.quickstart_table", "created": { @@ -2695,6 +2715,7 @@ "catalog": "system", "unity_schema": "quickstart_schema" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data/system/quickstart_schema", "name": "quickstart_schema", "description": "A new Unity Catalog schema called quickstart_schema" } @@ -2828,6 +2849,7 @@ "updated_by": "abc@acryl.io", "updated_at": "2022-10-19 13:27:29.633000" }, + "externalUrl": "https://dummy.cloud.databricks.com/explore/data/system/quickstart_schema/quickstart_table", "name": "quickstart_table", "qualifiedName": "system.quickstart_schema.quickstart_table", "created": { diff --git a/metadata-ingestion/tests/unit/test_unity_catalog_config.py b/metadata-ingestion/tests/unit/test_unity_catalog_config.py index d310ea8ce57d4..4be6f60171844 100644 --- a/metadata-ingestion/tests/unit/test_unity_catalog_config.py +++ b/metadata-ingestion/tests/unit/test_unity_catalog_config.py @@ -8,36 +8,6 @@ FROZEN_TIME = datetime.fromisoformat("2023-01-01 00:00:00+00:00") -def test_only_ingest_assigned_metastore(): - _ = UnityCatalogSourceConfig.parse_obj( - { - "token": "token", - "workspace_url": "https://workspace_url", - "metastore_id_pattern": {"allow": ["[a-zA-Z]+"]}, - } - ) - _ = UnityCatalogSourceConfig.parse_obj( - { - "token": "token", - "workspace_url": "https://workspace_url", - "only_ingest_assigned_metastore": True, - } - ) - - with pytest.raises( - ValueError, - match="metastore_id_pattern cannot be set when only_ingest_assigned_metastore is specified.", - ): - UnityCatalogSourceConfig.parse_obj( - { - "token": "token", - "workspace_url": "https://workspace_url", - "metastore_id_pattern": {"allow": ["[a-zA-Z]+"]}, - "only_ingest_assigned_metastore": True, - } - ) - - @freeze_time(FROZEN_TIME) def test_within_thirty_days(): config = UnityCatalogSourceConfig.parse_obj( From 5e45db9e60254b1d68c4d924a41085ee8498616c Mon Sep 17 00:00:00 2001 From: matthew-coudert-cko <113023884+matthew-coudert-cko@users.noreply.github.com> Date: Thu, 15 Jun 2023 08:16:25 +0100 Subject: [PATCH 018/222] docs(airflow): add docs on custom operators (#7913) Co-authored-by: John Joyce --- docs/lineage/airflow.md | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/docs/lineage/airflow.md b/docs/lineage/airflow.md index da7d4a625cb6f..ef4071f89c585 100644 --- a/docs/lineage/airflow.md +++ b/docs/lineage/airflow.md @@ -69,14 +69,41 @@ lazy_load_plugins = False ### How to validate installation -1. Go and check in Airflow at Admin -> Plugins menu if you can see the Datahub plugin +1. Go and check in Airflow at Admin -> Plugins menu if you can see the DataHub plugin 2. Run an Airflow DAG. In the task logs, you should see Datahub related log messages like: ``` -Emitting Datahub ... +Emitting DataHub ... ``` -## Using Datahub's Airflow lineage backend (deprecated) +### Emitting lineage via a custom operator to the Airflow Plugin + +If you have created a custom Airflow operator [docs](https://airflow.apache.org/docs/apache-airflow/stable/howto/custom-operator.html) that inherits from the BaseOperator class, +when overriding the `execute` function, set inlets and outlets via `context['ti'].task.inlets` and `context['ti'].task.outlets`. +The DataHub Airflow plugin will then pick up those inlets and outlets after the task runs. + + + +```python +class DbtOperator(BaseOperator): + ... + + def execute(self, context): + # do something + inlets, outlets = self._get_lineage() + # inlets/outlets are lists of either datahub_provider.entities.Dataset or datahub_provider.entities.Urn + context['ti'].task.inlets = self.inlets + context['ti'].task.outlets = self.outlets + + def _get_lineage(self): + # Do some processing to get inlets/outlets + + return inlets, outlets +``` + +If you override the `pre_execute` and `post_execute` function, ensure they include the `@prepare_lineage` and `@apply_lineage` decorators respectively. [source](https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/lineage.html#lineage) + +## Using DataHub's Airflow lineage backend (deprecated) :::caution @@ -145,6 +172,7 @@ Take a look at this sample DAG: In order to use this example, you must first configure the Datahub hook. Like in ingestion, we support a Datahub REST hook and a Kafka-based hook. See step 1 above for details. + ## Debugging ### Incorrect URLs From b0f686b05fbc6eec8e26a7b4078b769b9d1f3dc3 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Thu, 15 Jun 2023 12:08:01 +0100 Subject: [PATCH 019/222] chore(release): update datahub upgrade docs (#8228) --- docs/how/updating-datahub.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 8d8f24c4aed2c..4f8e193466778 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -12,6 +12,18 @@ This file documents any backwards-incompatible changes in DataHub and assists pe ### Other notable Changes +## 0.10.4 + +### Breaking Changes + +### Potential Downtime + +### Deprecations +- #8045: With the introduction of custom ownership types, the `Owner` aspect has been updated where the `type` field is deprecated in favor of a new field `typeUrn`. This latter field is an urn reference to the new OwnershipType entity. GraphQL endpoints have been updated to use the new field. For pre-existing ownership aspect records, DataHub now has logic to map the old field to the new field. + +### Other notable Changes +- #8191: Updates GMS's health check endpoint to account for its dependency on external components. Notably, at this time, elasticsearch. This means that DataHub operators can now use GMS health status more reliably. + ## 0.10.3 ### Breaking Changes From 25fa4ebbef3c3138aef11eba01e0eefb90e4a618 Mon Sep 17 00:00:00 2001 From: mohdsiddique Date: Thu, 15 Jun 2023 16:38:30 +0530 Subject: [PATCH 020/222] fix(ingestion/tableau): Remove unused field documentViewId (#8225) Co-authored-by: MohdSiddiqueBagwan --- .../src/datahub/ingestion/source/tableau_common.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py b/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py index 2a48b40891748..996d836b2f987 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py @@ -74,7 +74,6 @@ class MetadataQueryException(Exception): name path luid - documentViewId createdAt updatedAt tags { From 6ce0a248fd5c790a0dcdbcfe8428c87907e4094a Mon Sep 17 00:00:00 2001 From: RyanHolstien Date: Thu, 15 Jun 2023 06:17:13 -0500 Subject: [PATCH 021/222] feat(ui): create fast path for immediate processing of ui sourced changes (#8200) Co-authored-by: david-leifker <114954101+david-leifker@users.noreply.github.com> --- .../graphql/featureflags/FeatureFlags.java | 2 + .../UpdateDeprecationResolver.java | 15 +- .../domain/CreateDomainResolver.java | 15 +- .../resolvers/domain/SetDomainResolver.java | 15 +- .../resolvers/domain/UnsetDomainResolver.java | 16 +- .../resolvers/embed/UpdateEmbedResolver.java | 13 +- .../glossary/CreateGlossaryNodeResolver.java | 17 +- .../glossary/CreateGlossaryTermResolver.java | 17 +- ...ncelIngestionExecutionRequestResolver.java | 30 +- ...eateIngestionExecutionRequestResolver.java | 27 +- .../CreateTestConnectionRequestResolver.java | 19 +- .../ingest/secret/CreateSecretResolver.java | 21 +- .../source/UpsertIngestionSourceResolver.java | 25 +- .../resolvers/mutate/MutationUtils.java | 49 +- .../mutate/UpdateUserSettingResolver.java | 2 +- .../resolvers/mutate/util/DeleteUtils.java | 2 +- .../mutate/util/DeprecationUtils.java | 2 +- .../resolvers/mutate/util/DomainUtils.java | 2 +- .../resolvers/mutate/util/LabelUtils.java | 16 +- .../resolvers/mutate/util/OwnerUtils.java | 4 +- .../operation/ReportOperationResolver.java | 18 +- .../policy/UpsertPolicyResolver.java | 27 +- .../resolvers/tag/CreateTagResolver.java | 18 +- .../resolvers/tag/SetTagColorResolver.java | 15 +- .../resolvers/test/CreateTestResolver.java | 22 +- .../resolvers/test/UpdateTestResolver.java | 19 +- .../user/UpdateUserStatusResolver.java | 10 +- .../types/corpgroup/CorpGroupType.java | 14 +- .../graphql/types/corpuser/CorpUserType.java | 17 +- .../BatchUpdateSoftDeletedResolverTest.java | 36 +- .../BatchUpdateDeprecationResolverTest.java | 37 +- .../UpdateDeprecationResolverTest.java | 27 +- .../domain/BatchSetDomainResolverTest.java | 45 +- .../domain/CreateDomainResolverTest.java | 5 +- .../domain/SetDomainResolverTest.java | 27 +- .../domain/UnsetDomainResolverTest.java | 27 +- .../embed/UpdateEmbedResolverTest.java | 26 +- .../CreateGlossaryNodeResolverTest.java | 25 +- .../CreateGlossaryTermResolverTest.java | 25 +- .../glossary/UpdateNameResolverTest.java | 30 +- .../UpdateParentNodeResolverTest.java | 21 +- ...IngestionExecutionRequestResolverTest.java | 3 +- ...IngestionExecutionRequestResolverTest.java | 3 +- ...eateTestConnectionRequestResolverTest.java | 3 +- .../secret/CreateSecretResolverTest.java | 3 +- .../UpsertIngestionSourceResolverTest.java | 21 +- .../mutate/UpdateUserSettingResolverTest.java | 12 +- .../ReportOperationResolverTest.java | 16 +- .../resolvers/tag/AddTagsResolverTest.java | 29 +- .../tag/BatchAddTagsResolverTest.java | 36 +- .../tag/BatchRemoveTagsResolverTest.java | 29 +- .../resolvers/tag/CreateTagResolverTest.java | 19 +- .../tag/SetTagColorResolverTest.java | 16 +- .../test/CreateTestResolverTest.java | 39 +- .../test/UpdateTestResolverTest.java | 36 +- .../graphql/utils/MutationsUtilsTest.java | 25 + .../java/com/linkedin/metadata/Constants.java | 5 + .../metadata/config/PreProcessHooks.java | 9 + .../metadata/entity/EntityService.java | 66 +-- .../metadata/graph}/GraphIndexUtils.java | 3 +- .../service/UpdateIndicesService.java | 554 ++++++++++++++++++ .../linkedin/metadata/AspectUtilsTest.java | 6 +- .../metadata/ESSampleDataFixture.java | 6 +- .../metadata/ESSearchLineageFixture.java | 6 +- .../entity/AspectMigrationsDaoTest.java | 2 + .../CassandraAspectMigrationsDaoTest.java | 8 +- .../entity/CassandraEntityServiceTest.java | 8 +- .../entity/DeleteEntityServiceTest.java | 9 +- .../entity/EbeanAspectMigrationsDaoTest.java | 8 +- .../entity/EbeanEntityServiceTest.java | 8 +- .../metadata/entity/EntityServiceTest.java | 32 +- .../CassandraTimelineServiceTest.java | 6 +- .../timeline/EbeanTimelineServiceTest.java | 6 +- .../timeline/TimelineServiceTest.java | 4 + .../kafka/MaeConsumerApplication.java | 5 +- .../kafka/hook/UpdateIndicesHook.java | 541 +---------------- .../kafka/hook/GraphIndexUtilsTest.java | 3 +- .../kafka/hook/UpdateIndicesHookTest.java | 48 +- .../spring/MCLSpringTestConfiguration.java | 3 +- .../factory/entity/EntityServiceFactory.java | 9 +- .../indices/UpdateIndicesServiceFactory.java | 24 + .../src/main/resources/application.yml | 2 + .../java/entities/EntitiesControllerTest.java | 10 +- .../src/test/java/mock/MockEntityService.java | 7 +- .../linkedin/metadata/utils/PegasusUtils.java | 35 ++ 85 files changed, 1271 insertions(+), 1252 deletions(-) create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/MutationsUtilsTest.java create mode 100644 metadata-io/src/main/java/com/linkedin/metadata/config/PreProcessHooks.java rename {metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook => metadata-io/src/main/java/com/linkedin/metadata/graph}/GraphIndexUtils.java (98%) create mode 100644 metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java create mode 100644 metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/update/indices/UpdateIndicesServiceFactory.java diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java index 82a6e53f3009d..0287e6656a229 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql.featureflags; +import com.linkedin.metadata.config.PreProcessHooks; import lombok.Data; @@ -10,4 +11,5 @@ public class FeatureFlags { private boolean pointInTimeCreationEnabled = false; private boolean alwaysEmitChangeLog = false; private boolean readOnlyModeEnabled = false; + private PreProcessHooks preProcessHooks; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/deprecation/UpdateDeprecationResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/deprecation/UpdateDeprecationResolver.java index 2dfc564b3e42d..b132f74adc0f0 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/deprecation/UpdateDeprecationResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/deprecation/UpdateDeprecationResolver.java @@ -12,11 +12,8 @@ import com.linkedin.datahub.graphql.generated.UpdateDeprecationInput; import com.linkedin.datahub.graphql.resolvers.AuthUtils; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; @@ -27,6 +24,7 @@ import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; +import static com.linkedin.metadata.Constants.*; /** @@ -59,19 +57,14 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw try { Deprecation deprecation = (Deprecation) getAspectFromEntity( entityUrn.toString(), - Constants.DEPRECATION_ASPECT_NAME, + DEPRECATION_ASPECT_NAME, _entityService, new Deprecation()); updateDeprecation(deprecation, input, context); // Create the Deprecation aspect - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(entityUrn); - proposal.setEntityType(entityUrn.getEntityType()); - proposal.setAspectName(Constants.DEPRECATION_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(deprecation)); - proposal.setChangeType(ChangeType.UPSERT); - _entityClient.ingestProposal(proposal, context.getAuthentication()); + final MetadataChangeProposal proposal = buildMetadataChangeProposalWithUrn(entityUrn, DEPRECATION_ASPECT_NAME, deprecation); + _entityClient.ingestProposal(proposal, context.getAuthentication(), false); return true; } catch (Exception e) { log.error("Failed to update Deprecation for resource with entity urn {}: {}", entityUrn, e.getMessage()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java index d89f52bb7a03e..39aa1ea28da20 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java @@ -12,9 +12,7 @@ import com.linkedin.datahub.graphql.resolvers.mutate.util.OwnerUtils; import com.linkedin.domain.DomainProperties; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.key.DomainKey; import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.GenericRecordUtils; @@ -28,6 +26,8 @@ import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; import static com.linkedin.datahub.graphql.resolvers.mutate.util.OwnerUtils.*; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; +import static com.linkedin.metadata.Constants.*; /** @@ -60,19 +60,16 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws final String id = input.getId() != null ? input.getId() : UUID.randomUUID().toString(); key.setId(id); - if (_entityClient.exists(EntityKeyUtils.convertEntityKeyToUrn(key, Constants.DOMAIN_ENTITY_NAME), context.getAuthentication())) { + if (_entityClient.exists(EntityKeyUtils.convertEntityKeyToUrn(key, DOMAIN_ENTITY_NAME), context.getAuthentication())) { throw new IllegalArgumentException("This Domain already exists!"); } // Create the MCP - final MetadataChangeProposal proposal = new MetadataChangeProposal(); + final MetadataChangeProposal proposal = buildMetadataChangeProposalWithKey(key, DOMAIN_ENTITY_NAME, + DOMAIN_PROPERTIES_ASPECT_NAME, mapDomainProperties(input, context)); proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); - proposal.setEntityType(Constants.DOMAIN_ENTITY_NAME); - proposal.setAspectName(Constants.DOMAIN_PROPERTIES_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(mapDomainProperties(input, context))); - proposal.setChangeType(ChangeType.UPSERT); - String domainUrn = _entityClient.ingestProposal(proposal, context.getAuthentication()); + String domainUrn = _entityClient.ingestProposal(proposal, context.getAuthentication(), false); OwnershipType ownershipType = OwnershipType.TECHNICAL_OWNER; if (!_entityService.exists(UrnUtils.getUrn(mapOwnershipTypeToEntity(ownershipType.name())))) { log.warn("Technical owner does not exist, defaulting to None ownership."); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/SetDomainResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/SetDomainResolver.java index f1cfca6c19523..3936bfb793cd1 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/SetDomainResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/SetDomainResolver.java @@ -7,10 +7,7 @@ import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; import com.linkedin.domain.Domains; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; @@ -19,6 +16,7 @@ import lombok.extern.slf4j.Slf4j; import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; +import static com.linkedin.metadata.Constants.*; /** @@ -51,19 +49,14 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw try { Domains domains = (Domains) getAspectFromEntity( entityUrn.toString(), - Constants.DOMAINS_ASPECT_NAME, + DOMAINS_ASPECT_NAME, _entityService, new Domains()); setDomain(domains, domainUrn); // Create the Domains aspects - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(entityUrn); - proposal.setEntityType(entityUrn.getEntityType()); - proposal.setAspectName(Constants.DOMAINS_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(domains)); - proposal.setChangeType(ChangeType.UPSERT); - _entityClient.ingestProposal(proposal, context.getAuthentication()); + final MetadataChangeProposal proposal = buildMetadataChangeProposalWithUrn(entityUrn, DOMAINS_ASPECT_NAME, domains); + _entityClient.ingestProposal(proposal, context.getAuthentication(), false); return true; } catch (Exception e) { log.error("Failed to set Domain to resource with entity urn {}, domain urn {}: {}", entityUrn, domainUrn, e.getMessage()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/UnsetDomainResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/UnsetDomainResolver.java index ea664bd046595..82828cbac4477 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/UnsetDomainResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/UnsetDomainResolver.java @@ -7,10 +7,7 @@ import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; import com.linkedin.domain.Domains; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; @@ -20,6 +17,8 @@ import lombok.extern.slf4j.Slf4j; import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; +import static com.linkedin.metadata.Constants.*; + /** * Resolver used for removing the Domain associated with a Metadata Asset. Requires the EDIT_DOMAINS privilege for a particular asset. @@ -50,19 +49,14 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw try { Domains domains = (Domains) getAspectFromEntity( entityUrn.toString(), - Constants.DOMAINS_ASPECT_NAME, + DOMAINS_ASPECT_NAME, _entityService, new Domains()); unsetDomain(domains); // Create the Domains aspects - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(entityUrn); - proposal.setEntityType(entityUrn.getEntityType()); - proposal.setAspectName(Constants.DOMAINS_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(domains)); - proposal.setChangeType(ChangeType.UPSERT); - _entityClient.ingestProposal(proposal, context.getAuthentication()); + final MetadataChangeProposal proposal = buildMetadataChangeProposalWithUrn(entityUrn, DOMAINS_ASPECT_NAME, domains); + _entityClient.ingestProposal(proposal, context.getAuthentication(), false); return true; } catch (Exception e) { log.error("Failed to unset Domains for resource with entity urn {}: {}", entityUrn, e.getMessage()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/embed/UpdateEmbedResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/embed/UpdateEmbedResolver.java index 3d4a55b0cdbb6..86b8eb5564152 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/embed/UpdateEmbedResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/embed/UpdateEmbedResolver.java @@ -9,10 +9,7 @@ import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.generated.UpdateEmbedInput; import com.linkedin.datahub.graphql.resolvers.mutate.util.EmbedUtils; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; @@ -23,6 +20,7 @@ import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; +import static com.linkedin.metadata.Constants.*; /** @@ -53,18 +51,13 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw try { final Embed embed = (Embed) getAspectFromEntity( entityUrn.toString(), - Constants.EMBED_ASPECT_NAME, + EMBED_ASPECT_NAME, _entityService, new Embed()); updateEmbed(embed, input); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(entityUrn); - proposal.setEntityType(entityUrn.getEntityType()); - proposal.setAspectName(Constants.EMBED_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(embed)); - proposal.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal = buildMetadataChangeProposalWithUrn(entityUrn, EMBED_ASPECT_NAME, embed); _entityService.ingestProposal( proposal, new AuditStamp().setActor(UrnUtils.getUrn(context.getActorUrn())).setTime(System.currentTimeMillis()), diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolver.java index ee5b896043d62..cc0ab4e03a4e8 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolver.java @@ -12,13 +12,10 @@ import com.linkedin.datahub.graphql.resolvers.mutate.util.GlossaryUtils; import com.linkedin.datahub.graphql.resolvers.mutate.util.OwnerUtils; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.glossary.GlossaryNodeInfo; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.key.GlossaryNodeKey; import com.linkedin.metadata.utils.EntityKeyUtils; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; @@ -31,6 +28,8 @@ import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; import static com.linkedin.datahub.graphql.resolvers.mutate.util.OwnerUtils.*; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; +import static com.linkedin.metadata.Constants.*; @Slf4j @@ -55,18 +54,14 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws final String id = input.getId() != null ? input.getId() : UUID.randomUUID().toString(); key.setName(id); - if (_entityClient.exists(EntityKeyUtils.convertEntityKeyToUrn(key, Constants.GLOSSARY_NODE_ENTITY_NAME), context.getAuthentication())) { + if (_entityClient.exists(EntityKeyUtils.convertEntityKeyToUrn(key, GLOSSARY_NODE_ENTITY_NAME), context.getAuthentication())) { throw new IllegalArgumentException("This Glossary Node already exists!"); } - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); - proposal.setEntityType(Constants.GLOSSARY_NODE_ENTITY_NAME); - proposal.setAspectName(Constants.GLOSSARY_NODE_INFO_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(mapGlossaryNodeInfo(input))); - proposal.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal = buildMetadataChangeProposalWithKey(key, GLOSSARY_NODE_ENTITY_NAME, + GLOSSARY_NODE_INFO_ASPECT_NAME, mapGlossaryNodeInfo(input)); - String glossaryNodeUrn = _entityClient.ingestProposal(proposal, context.getAuthentication()); + String glossaryNodeUrn = _entityClient.ingestProposal(proposal, context.getAuthentication(), false); OwnershipType ownershipType = OwnershipType.TECHNICAL_OWNER; if (!_entityService.exists(UrnUtils.getUrn(mapOwnershipTypeToEntity(ownershipType.name())))) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryTermResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryTermResolver.java index 6db79e5c347ea..e454fe3aa4f67 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryTermResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryTermResolver.java @@ -12,13 +12,10 @@ import com.linkedin.datahub.graphql.resolvers.mutate.util.GlossaryUtils; import com.linkedin.datahub.graphql.resolvers.mutate.util.OwnerUtils; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.glossary.GlossaryTermInfo; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.key.GlossaryTermKey; import com.linkedin.metadata.utils.EntityKeyUtils; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; @@ -31,6 +28,8 @@ import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; import static com.linkedin.datahub.graphql.resolvers.mutate.util.OwnerUtils.*; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; +import static com.linkedin.metadata.Constants.*; @Slf4j @@ -55,18 +54,14 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws final String id = input.getId() != null ? input.getId() : UUID.randomUUID().toString(); key.setName(id); - if (_entityClient.exists(EntityKeyUtils.convertEntityKeyToUrn(key, Constants.GLOSSARY_TERM_ENTITY_NAME), context.getAuthentication())) { + if (_entityClient.exists(EntityKeyUtils.convertEntityKeyToUrn(key, GLOSSARY_TERM_ENTITY_NAME), context.getAuthentication())) { throw new IllegalArgumentException("This Glossary Term already exists!"); } - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); - proposal.setEntityType(Constants.GLOSSARY_TERM_ENTITY_NAME); - proposal.setAspectName(Constants.GLOSSARY_TERM_INFO_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(mapGlossaryTermInfo(input))); - proposal.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal = buildMetadataChangeProposalWithKey(key, GLOSSARY_TERM_ENTITY_NAME, + GLOSSARY_TERM_INFO_ASPECT_NAME, mapGlossaryTermInfo(input)); - String glossaryTermUrn = _entityClient.ingestProposal(proposal, context.getAuthentication()); + String glossaryTermUrn = _entityClient.ingestProposal(proposal, context.getAuthentication(), false); OwnershipType ownershipType = OwnershipType.TECHNICAL_OWNER; if (!_entityService.exists(UrnUtils.getUrn(mapOwnershipTypeToEntity(ownershipType.name())))) { log.warn("Technical owner does not exist, defaulting to None ownership."); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CancelIngestionExecutionRequestResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CancelIngestionExecutionRequestResolver.java index 750b13d76f6a5..7f9cb6176989f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CancelIngestionExecutionRequestResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CancelIngestionExecutionRequestResolver.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableSet; import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.SetMode; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.exception.AuthorizationException; @@ -13,11 +14,8 @@ import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.execution.ExecutionRequestSignal; import com.linkedin.ingestion.DataHubIngestionSourceInfo; -import com.linkedin.metadata.Constants; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; @@ -25,6 +23,9 @@ import java.util.concurrent.CompletableFuture; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; +import static com.linkedin.metadata.Constants.*; + /** * Cancels a requested ingestion execution by emitting a KILL signal. @@ -51,23 +52,19 @@ public CompletableFuture get(final DataFetchingEnvironment environment) bindArgument(environment.getArgument("input"), CancelIngestionExecutionRequestInput.class); try { - - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(input.getExecutionRequestUrn())); - final Urn ingestionSourceUrn = Urn.createFromString(input.getIngestionSourceUrn()); final Map response = - _entityClient.batchGetV2(Constants.INGESTION_SOURCE_ENTITY_NAME, ImmutableSet.of(ingestionSourceUrn), - ImmutableSet.of(Constants.INGESTION_INFO_ASPECT_NAME), context.getAuthentication()); + _entityClient.batchGetV2(INGESTION_SOURCE_ENTITY_NAME, ImmutableSet.of(ingestionSourceUrn), + ImmutableSet.of(INGESTION_INFO_ASPECT_NAME), context.getAuthentication()); if (!response.containsKey(ingestionSourceUrn)) { throw new DataHubGraphQLException( - String.format("Failed to find ingestion source with urn %s", ingestionSourceUrn.toString()), + String.format("Failed to find ingestion source with urn %s", ingestionSourceUrn), DataHubGraphQLErrorCode.BAD_REQUEST); } final EnvelopedAspect envelopedInfo = - response.get(ingestionSourceUrn).getAspects().get(Constants.INGESTION_INFO_ASPECT_NAME); + response.get(ingestionSourceUrn).getAspects().get(INGESTION_INFO_ASPECT_NAME); final DataHubIngestionSourceInfo ingestionSourceInfo = new DataHubIngestionSourceInfo(envelopedInfo.getValue().data()); // Build the arguments map. @@ -78,14 +75,11 @@ public CompletableFuture get(final DataFetchingEnvironment environment) .setTime(System.currentTimeMillis()) .setActor(Urn.createFromString(context.getActorUrn())) ); - proposal.setEntityType(Constants.EXECUTION_REQUEST_ENTITY_NAME); - proposal.setAspectName(Constants.EXECUTION_REQUEST_SIGNAL_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(execSignal)); - proposal.setChangeType(ChangeType.UPSERT); - - return _entityClient.ingestProposal(proposal, context.getAuthentication()); + final MetadataChangeProposal proposal = buildMetadataChangeProposalWithUrn(UrnUtils.getUrn( + input.getExecutionRequestUrn()), EXECUTION_REQUEST_SIGNAL_ASPECT_NAME, execSignal); + return _entityClient.ingestProposal(proposal, context.getAuthentication(), false); } catch (Exception e) { - throw new RuntimeException(String.format("Failed to submit cancel signal %s", input.toString()), e); + throw new RuntimeException(String.format("Failed to submit cancel signal %s", input), e); } } throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolver.java index f173796409bc9..e5064e6620526 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolver.java @@ -13,15 +13,12 @@ import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.execution.ExecutionRequestInput; import com.linkedin.execution.ExecutionRequestSource; import com.linkedin.ingestion.DataHubIngestionSourceInfo; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.config.IngestionConfiguration; import com.linkedin.metadata.key.ExecutionRequestKey; import com.linkedin.metadata.utils.EntityKeyUtils; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.metadata.utils.IngestionUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetcher; @@ -34,6 +31,8 @@ import org.json.JSONObject; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; +import static com.linkedin.metadata.Constants.*; /** @@ -67,20 +66,17 @@ public CompletableFuture get(final DataFetchingEnvironment environment) bindArgument(environment.getArgument("input"), CreateIngestionExecutionRequestInput.class); try { - - final MetadataChangeProposal proposal = new MetadataChangeProposal(); final ExecutionRequestKey key = new ExecutionRequestKey(); final UUID uuid = UUID.randomUUID(); final String uuidStr = uuid.toString(); key.setId(uuidStr); - final Urn executionRequestUrn = EntityKeyUtils.convertEntityKeyToUrn(key, Constants.EXECUTION_REQUEST_ENTITY_NAME); - proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); + final Urn executionRequestUrn = EntityKeyUtils.convertEntityKeyToUrn(key, EXECUTION_REQUEST_ENTITY_NAME); // Fetch the original ingestion source final Urn ingestionSourceUrn = Urn.createFromString(input.getIngestionSourceUrn()); final Map response = - _entityClient.batchGetV2(Constants.INGESTION_SOURCE_ENTITY_NAME, ImmutableSet.of(ingestionSourceUrn), - ImmutableSet.of(Constants.INGESTION_INFO_ASPECT_NAME), context.getAuthentication()); + _entityClient.batchGetV2(INGESTION_SOURCE_ENTITY_NAME, ImmutableSet.of(ingestionSourceUrn), + ImmutableSet.of(INGESTION_INFO_ASPECT_NAME), context.getAuthentication()); if (!response.containsKey(ingestionSourceUrn)) { throw new DataHubGraphQLException( @@ -88,7 +84,7 @@ public CompletableFuture get(final DataFetchingEnvironment environment) DataHubGraphQLErrorCode.BAD_REQUEST); } - final EnvelopedAspect envelopedInfo = response.get(ingestionSourceUrn).getAspects().get(Constants.INGESTION_INFO_ASPECT_NAME); + final EnvelopedAspect envelopedInfo = response.get(ingestionSourceUrn).getAspects().get(INGESTION_INFO_ASPECT_NAME); final DataHubIngestionSourceInfo ingestionSourceInfo = new DataHubIngestionSourceInfo(envelopedInfo.getValue().data()); if (!ingestionSourceInfo.getConfig().hasRecipe()) { @@ -124,14 +120,11 @@ public CompletableFuture get(final DataFetchingEnvironment environment) arguments.put(DEBUG_MODE_ARG_NAME, debugMode); execInput.setArgs(new StringMap(arguments)); - proposal.setEntityType(Constants.EXECUTION_REQUEST_ENTITY_NAME); - proposal.setAspectName(Constants.EXECUTION_REQUEST_INPUT_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(execInput)); - proposal.setChangeType(ChangeType.UPSERT); - - return _entityClient.ingestProposal(proposal, context.getAuthentication()); + final MetadataChangeProposal proposal = buildMetadataChangeProposalWithKey(key, + EXECUTION_REQUEST_ENTITY_NAME, EXECUTION_REQUEST_INPUT_ASPECT_NAME, execInput); + return _entityClient.ingestProposal(proposal, context.getAuthentication(), false); } catch (Exception e) { - throw new RuntimeException(String.format("Failed to create new ingestion execution request %s", input.toString()), e); + throw new RuntimeException(String.format("Failed to create new ingestion execution request %s", input), e); } } throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolver.java index 07aad0fc75948..8f4d538ca67ec 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolver.java @@ -7,14 +7,11 @@ import com.linkedin.datahub.graphql.generated.CreateTestConnectionRequestInput; import com.linkedin.datahub.graphql.resolvers.ingest.IngestionAuthUtils; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.execution.ExecutionRequestInput; import com.linkedin.execution.ExecutionRequestSource; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.config.IngestionConfiguration; import com.linkedin.metadata.key.ExecutionRequestKey; import com.linkedin.metadata.utils.EntityKeyUtils; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.metadata.utils.IngestionUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetcher; @@ -26,6 +23,8 @@ import java.util.concurrent.CompletableFuture; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; +import static com.linkedin.metadata.Constants.*; /** @@ -61,14 +60,11 @@ public CompletableFuture get(final DataFetchingEnvironment environment) bindArgument(environment.getArgument("input"), CreateTestConnectionRequestInput.class); try { - - final MetadataChangeProposal proposal = new MetadataChangeProposal(); final ExecutionRequestKey key = new ExecutionRequestKey(); final UUID uuid = UUID.randomUUID(); final String uuidStr = uuid.toString(); key.setId(uuidStr); - final Urn executionRequestUrn = EntityKeyUtils.convertEntityKeyToUrn(key, Constants.EXECUTION_REQUEST_ENTITY_NAME); - proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); + final Urn executionRequestUrn = EntityKeyUtils.convertEntityKeyToUrn(key, EXECUTION_REQUEST_ENTITY_NAME); final ExecutionRequestInput execInput = new ExecutionRequestInput(); execInput.setTask(TEST_CONNECTION_TASK_NAME); @@ -83,12 +79,9 @@ public CompletableFuture get(final DataFetchingEnvironment environment) } execInput.setArgs(new StringMap(arguments)); - proposal.setEntityType(Constants.EXECUTION_REQUEST_ENTITY_NAME); - proposal.setAspectName(Constants.EXECUTION_REQUEST_INPUT_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(execInput)); - proposal.setChangeType(ChangeType.UPSERT); - - return _entityClient.ingestProposal(proposal, context.getAuthentication()); + final MetadataChangeProposal proposal = buildMetadataChangeProposalWithKey(key, EXECUTION_REQUEST_ENTITY_NAME, + EXECUTION_REQUEST_INPUT_ASPECT_NAME, execInput); + return _entityClient.ingestProposal(proposal, context.getAuthentication(), false); } catch (Exception e) { throw new RuntimeException(String.format("Failed to create new test ingestion connection request %s", input.toString()), e); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/CreateSecretResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/CreateSecretResolver.java index 02821cb67c898..e1745031d9dae 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/CreateSecretResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/CreateSecretResolver.java @@ -8,12 +8,9 @@ import com.linkedin.datahub.graphql.generated.CreateSecretInput; import com.linkedin.datahub.graphql.resolvers.ingest.IngestionAuthUtils; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.key.DataHubSecretKey; import com.linkedin.metadata.secret.SecretService; import com.linkedin.metadata.utils.EntityKeyUtils; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.secret.DataHubSecretValue; import graphql.schema.DataFetcher; @@ -21,6 +18,9 @@ import java.util.concurrent.CompletableFuture; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; +import static com.linkedin.metadata.Constants.*; + /** * Creates an encrypted DataHub secret. Uses AES symmetric encryption / decryption. Requires the MANAGE_SECRETS privilege. @@ -48,15 +48,11 @@ public CompletableFuture get(final DataFetchingEnvironment environment) if (IngestionAuthUtils.canManageSecrets(context)) { try { - - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - // Create the Ingestion source key --> use the display name as a unique id to ensure it's not duplicated. final DataHubSecretKey key = new DataHubSecretKey(); key.setId(input.getName()); - proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); - if (_entityClient.exists(EntityKeyUtils.convertEntityKeyToUrn(key, Constants.SECRETS_ENTITY_NAME), context.getAuthentication())) { + if (_entityClient.exists(EntityKeyUtils.convertEntityKeyToUrn(key, SECRETS_ENTITY_NAME), context.getAuthentication())) { throw new IllegalArgumentException("This Secret already exists!"); } @@ -67,12 +63,9 @@ public CompletableFuture get(final DataFetchingEnvironment environment) value.setDescription(input.getDescription(), SetMode.IGNORE_NULL); value.setCreated(new AuditStamp().setActor(UrnUtils.getUrn(context.getActorUrn())).setTime(System.currentTimeMillis())); - proposal.setEntityType(Constants.SECRETS_ENTITY_NAME); - proposal.setAspectName(Constants.SECRET_VALUE_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(value)); - proposal.setChangeType(ChangeType.UPSERT); - - return _entityClient.ingestProposal(proposal, context.getAuthentication()); + final MetadataChangeProposal proposal = buildMetadataChangeProposalWithKey(key, SECRETS_ENTITY_NAME, + SECRET_VALUE_ASPECT_NAME, value); + return _entityClient.ingestProposal(proposal, context.getAuthentication(), false); } catch (Exception e) { throw new RuntimeException(String.format("Failed to create new secret with name %s", input.getName()), e); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolver.java index 41e786927c01a..2ce394ad5ba84 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolver.java @@ -10,13 +10,10 @@ import com.linkedin.datahub.graphql.generated.UpdateIngestionSourceScheduleInput; import com.linkedin.datahub.graphql.resolvers.ingest.IngestionAuthUtils; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.ingestion.DataHubIngestionSourceConfig; import com.linkedin.ingestion.DataHubIngestionSourceInfo; import com.linkedin.ingestion.DataHubIngestionSourceSchedule; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.key.DataHubIngestionSourceKey; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; @@ -28,6 +25,8 @@ import java.util.concurrent.CompletableFuture; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; +import static com.linkedin.metadata.Constants.*; /** @@ -53,12 +52,13 @@ public CompletableFuture get(final DataFetchingEnvironment environment) final Optional ingestionSourceUrn = Optional.ofNullable(environment.getArgument("urn")); final UpdateIngestionSourceInput input = bindArgument(environment.getArgument("input"), UpdateIngestionSourceInput.class); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - + // Create the policy info. + final DataHubIngestionSourceInfo info = mapIngestionSourceInfo(input); + final MetadataChangeProposal proposal; if (ingestionSourceUrn.isPresent()) { // Update existing ingestion source try { - proposal.setEntityUrn(Urn.createFromString(ingestionSourceUrn.get())); + proposal = buildMetadataChangeProposalWithUrn(Urn.createFromString(ingestionSourceUrn.get()), INGESTION_INFO_ASPECT_NAME, info); } catch (URISyntaxException e) { throw new DataHubGraphQLException( String.format("Malformed urn %s provided.", ingestionSourceUrn.get()), @@ -69,22 +69,13 @@ public CompletableFuture get(final DataFetchingEnvironment environment) // Since we are creating a new Ingestion Source, we need to generate a unique UUID. final UUID uuid = UUID.randomUUID(); final String uuidStr = uuid.toString(); - - // Create the Ingestion source key final DataHubIngestionSourceKey key = new DataHubIngestionSourceKey(); key.setId(uuidStr); - proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); + proposal = buildMetadataChangeProposalWithKey(key, INGESTION_SOURCE_ENTITY_NAME, INGESTION_INFO_ASPECT_NAME, info); } - // Create the policy info. - final DataHubIngestionSourceInfo info = mapIngestionSourceInfo(input); - proposal.setEntityType(Constants.INGESTION_SOURCE_ENTITY_NAME); - proposal.setAspectName(Constants.INGESTION_INFO_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(info)); - proposal.setChangeType(ChangeType.UPSERT); - try { - return _entityClient.ingestProposal(proposal, context.getAuthentication()); + return _entityClient.ingestProposal(proposal, context.getAuthentication(), false); } catch (Exception e) { throw new RuntimeException(String.format("Failed to perform update against ingestion source with urn %s", input.toString()), e); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MutationUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MutationUtils.java index 3cc64aed7f562..0cf9acd62f736 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MutationUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MutationUtils.java @@ -3,11 +3,13 @@ import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.StringMap; import com.linkedin.datahub.graphql.generated.SubResourceType; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.mxe.SystemMetadata; import com.linkedin.schema.EditableSchemaFieldInfo; import com.linkedin.schema.EditableSchemaFieldInfoArray; import com.linkedin.schema.EditableSchemaMetadata; @@ -16,6 +18,8 @@ import java.util.Optional; import lombok.extern.slf4j.Slf4j; +import static com.linkedin.metadata.Constants.*; + @Slf4j public class MutationUtils { @@ -24,22 +28,51 @@ public class MutationUtils { private MutationUtils() { } public static void persistAspect(Urn urn, String aspectName, RecordTemplate aspect, Urn actor, EntityService entityService) { - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(urn); - proposal.setEntityType(urn.getEntityType()); - proposal.setAspectName(aspectName); - proposal.setAspect(GenericRecordUtils.serializeAspect(aspect)); - proposal.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal = buildMetadataChangeProposalWithUrn(urn, aspectName, aspect); entityService.ingestProposal(proposal, getAuditStamp(actor), false); } - public static MetadataChangeProposal buildMetadataChangeProposal(Urn urn, String aspectName, RecordTemplate aspect, Urn actor, EntityService entityService) { + /** + * Only intended for use from GraphQL mutations, executes a different flow indicating a request sourced from the UI + * @param urn + * @param aspectName + * @param aspect + * @return + */ + public static MetadataChangeProposal buildMetadataChangeProposalWithUrn(Urn urn, String aspectName, RecordTemplate aspect) { final MetadataChangeProposal proposal = new MetadataChangeProposal(); proposal.setEntityUrn(urn); - proposal.setEntityType(urn.getEntityType()); + return setProposalProperties(proposal, urn.getEntityType(), aspectName, aspect); + } + + /** + * Only intended for use from GraphQL mutations, executes a different flow indicating a request sourced from the UI + * @param entityKey + * @param entityType + * @param aspectName + * @param aspect + * @return + */ + public static MetadataChangeProposal buildMetadataChangeProposalWithKey(RecordTemplate entityKey, String entityType, + String aspectName, RecordTemplate aspect) { + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(entityKey)); + return setProposalProperties(proposal, entityType, aspectName, aspect); + } + + private static MetadataChangeProposal setProposalProperties(MetadataChangeProposal proposal, + String entityType, String aspectName, RecordTemplate aspect) { + proposal.setEntityType(entityType); proposal.setAspectName(aspectName); proposal.setAspect(GenericRecordUtils.serializeAspect(aspect)); proposal.setChangeType(ChangeType.UPSERT); + + // Assumes proposal is generated first from the builder methods above so SystemMetadata is empty + SystemMetadata systemMetadata = new SystemMetadata(); + StringMap properties = new StringMap(); + properties.put(APP_SOURCE, UI_SOURCE); + systemMetadata.setProperties(properties); + proposal.setSystemMetadata(systemMetadata); return proposal; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateUserSettingResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateUserSettingResolver.java index 1531e0490ef04..8c1d32c470f44 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateUserSettingResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateUserSettingResolver.java @@ -54,7 +54,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw } MetadataChangeProposal proposal = - buildMetadataChangeProposal(actor, CORP_USER_SETTINGS_ASPECT_NAME, newSettings, actor, _entityService); + buildMetadataChangeProposalWithUrn(actor, CORP_USER_SETTINGS_ASPECT_NAME, newSettings); _entityService.ingestProposal(proposal, getAuditStamp(actor), false); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeleteUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeleteUtils.java index 84feaeacc688d..480357b89bb14 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeleteUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeleteUtils.java @@ -68,7 +68,7 @@ private static MetadataChangeProposal buildSoftDeleteProposal( entityService, new Status()); status.setRemoved(removed); - return buildMetadataChangeProposal(UrnUtils.getUrn(urnStr), Constants.STATUS_ASPECT_NAME, status, actor, entityService); + return buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(urnStr), Constants.STATUS_ASPECT_NAME, status); } private static void ingestChangeProposals(List changes, EntityService entityService, Urn actor) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeprecationUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeprecationUtils.java index 18832331b6652..615d76c650286 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeprecationUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeprecationUtils.java @@ -83,7 +83,7 @@ private static MetadataChangeProposal buildUpdateDeprecationProposal( // Note is required field in GMS. Set to empty string if not provided. deprecation.setNote(""); } - return buildMetadataChangeProposal(UrnUtils.getUrn(resource.getResourceUrn()), Constants.DEPRECATION_ASPECT_NAME, deprecation, actor, entityService); + return buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(resource.getResourceUrn()), Constants.DEPRECATION_ASPECT_NAME, deprecation); } private static void ingestChangeProposals(List changes, EntityService entityService, Urn actor) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java index 22d98d6004d84..213c08454d278 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java @@ -76,7 +76,7 @@ private static MetadataChangeProposal buildSetDomainProposal( newDomains.add(domainUrn); } domains.setDomains(newDomains); - return buildMetadataChangeProposal(UrnUtils.getUrn(resource.getResourceUrn()), Constants.DOMAINS_ASPECT_NAME, domains, actor, entityService); + return buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(resource.getResourceUrn()), Constants.DOMAINS_ASPECT_NAME, domains); } public static void validateDomain(Urn domainUrn, EntityService entityService) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LabelUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LabelUtils.java index 707648bd31ab7..1922a02fc1ca0 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LabelUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LabelUtils.java @@ -326,7 +326,7 @@ private static MetadataChangeProposal buildRemoveTagsToEntityProposal( tags.setTags(new TagAssociationArray()); } removeTagsIfExists(tags, tagUrns); - return buildMetadataChangeProposal(UrnUtils.getUrn(resource.getResourceUrn()), TAGS_ASPECT_NAME, tags, actor, entityService); + return buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(resource.getResourceUrn()), TAGS_ASPECT_NAME, tags); } private static MetadataChangeProposal buildRemoveTagsToSubResourceProposal( @@ -347,7 +347,7 @@ private static MetadataChangeProposal buildRemoveTagsToSubResourceProposal( editableFieldInfo.setGlobalTags(new GlobalTags()); } removeTagsIfExists(editableFieldInfo.getGlobalTags(), tagUrns); - return buildMetadataChangeProposal(UrnUtils.getUrn(resource.getResourceUrn()), EDITABLE_SCHEMA_METADATA, editableSchemaMetadata, actor, entityService); + return buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(resource.getResourceUrn()), EDITABLE_SCHEMA_METADATA, editableSchemaMetadata); } private static MetadataChangeProposal buildAddTagsToEntityProposal( @@ -363,7 +363,7 @@ private static MetadataChangeProposal buildAddTagsToEntityProposal( tags.setTags(new TagAssociationArray()); } addTagsIfNotExists(tags, tagUrns); - return buildMetadataChangeProposal(UrnUtils.getUrn(resource.getResourceUrn()), TAGS_ASPECT_NAME, tags, actor, entityService); + return buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(resource.getResourceUrn()), TAGS_ASPECT_NAME, tags); } private static MetadataChangeProposal buildAddTagsToSubResourceProposal( @@ -382,7 +382,7 @@ private static MetadataChangeProposal buildAddTagsToSubResourceProposal( } addTagsIfNotExists(editableFieldInfo.getGlobalTags(), tagUrns); - return buildMetadataChangeProposal(UrnUtils.getUrn(resource.getResourceUrn()), EDITABLE_SCHEMA_METADATA, editableSchemaMetadata, actor, entityService); + return buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(resource.getResourceUrn()), EDITABLE_SCHEMA_METADATA, editableSchemaMetadata); } private static void addTagsIfNotExists(GlobalTags tags, List tagUrns) throws URISyntaxException { @@ -457,7 +457,7 @@ private static MetadataChangeProposal buildAddTermsToEntityProposal( } addTermsIfNotExists(terms, termUrns); - return buildMetadataChangeProposal(UrnUtils.getUrn(resource.getResourceUrn()), GLOSSARY_TERM_ASPECT_NAME, terms, actor, entityService); + return buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(resource.getResourceUrn()), GLOSSARY_TERM_ASPECT_NAME, terms); } private static MetadataChangeProposal buildAddTermsToSubResourceProposal( @@ -478,7 +478,7 @@ private static MetadataChangeProposal buildAddTermsToSubResourceProposal( editableFieldInfo.getGlossaryTerms().setAuditStamp(getAuditStamp(actor)); addTermsIfNotExists(editableFieldInfo.getGlossaryTerms(), termUrns); - return buildMetadataChangeProposal(UrnUtils.getUrn(resource.getResourceUrn()), EDITABLE_SCHEMA_METADATA, editableSchemaMetadata, actor, entityService); + return buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(resource.getResourceUrn()), EDITABLE_SCHEMA_METADATA, editableSchemaMetadata); } private static MetadataChangeProposal buildRemoveTermsToEntityProposal( @@ -493,7 +493,7 @@ private static MetadataChangeProposal buildRemoveTermsToEntityProposal( terms.setAuditStamp(getAuditStamp(actor)); removeTermsIfExists(terms, termUrns); - return buildMetadataChangeProposal(UrnUtils.getUrn(resource.getResourceUrn()), GLOSSARY_TERM_ASPECT_NAME, terms, actor, entityService); + return buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(resource.getResourceUrn()), GLOSSARY_TERM_ASPECT_NAME, terms); } private static MetadataChangeProposal buildRemoveTermsToSubResourceProposal( @@ -511,7 +511,7 @@ private static MetadataChangeProposal buildRemoveTermsToSubResourceProposal( } removeTermsIfExists(editableFieldInfo.getGlossaryTerms(), termUrns); - return buildMetadataChangeProposal(UrnUtils.getUrn(resource.getResourceUrn()), EDITABLE_SCHEMA_METADATA, editableSchemaMetadata, actor, entityService); + return buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(resource.getResourceUrn()), EDITABLE_SCHEMA_METADATA, editableSchemaMetadata); } private static void addTermsIfNotExists(GlossaryTerms terms, List termUrns) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java index a062016ed5d7e..a08419b5226b4 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java @@ -78,7 +78,7 @@ private static MetadataChangeProposal buildAddOwnersProposal(List ow for (OwnerInput input : owners) { addOwner(ownershipAspect, UrnUtils.getUrn(input.getOwnerUrn()), input.getType(), UrnUtils.getUrn(input.getOwnershipTypeUrn())); } - return buildMetadataChangeProposal(resourceUrn, Constants.OWNERSHIP_ASPECT_NAME, ownershipAspect, actor, entityService); + return buildMetadataChangeProposalWithUrn(resourceUrn, Constants.OWNERSHIP_ASPECT_NAME, ownershipAspect); } public static MetadataChangeProposal buildRemoveOwnersProposal( @@ -93,7 +93,7 @@ public static MetadataChangeProposal buildRemoveOwnersProposal( new Ownership()); ownershipAspect.setLastModified(getAuditStamp(actor)); removeOwnersIfExists(ownershipAspect, ownerUrns, maybeOwnershipTypeUrn); - return buildMetadataChangeProposal(resourceUrn, Constants.OWNERSHIP_ASPECT_NAME, ownershipAspect, actor, entityService); + return buildMetadataChangeProposalWithUrn(resourceUrn, Constants.OWNERSHIP_ASPECT_NAME, ownershipAspect); } private static void addOwner(Ownership ownershipAspect, Urn ownerUrn, OwnershipType type, Urn ownershipUrn) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/operation/ReportOperationResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/operation/ReportOperationResolver.java index 18dcf1f8fefdf..c0fe697c6654c 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/operation/ReportOperationResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/operation/ReportOperationResolver.java @@ -18,10 +18,7 @@ import com.linkedin.datahub.graphql.generated.ReportOperationInput; import com.linkedin.datahub.graphql.generated.StringMapEntryInput; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.authorization.PoliciesConfig; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.timeseries.PartitionSpec; import com.linkedin.timeseries.PartitionType; @@ -35,6 +32,9 @@ import static com.linkedin.datahub.graphql.resolvers.AuthUtils.*; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; +import static com.linkedin.metadata.Constants.*; + /** * Resolver used for reporting Asset Operations @@ -44,7 +44,7 @@ public class ReportOperationResolver implements DataFetcher> { private static final List SUPPORTED_ENTITY_TYPES = ImmutableList.of( - Constants.DATASET_ENTITY_NAME + DATASET_ENTITY_NAME ); private final EntityClient _entityClient; @@ -67,13 +67,9 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw try { // Create an MCP to emit the operation - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(entityUrn); - proposal.setEntityType(entityUrn.getEntityType()); - proposal.setAspectName(Constants.OPERATION_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(mapOperation(input, context))); - proposal.setChangeType(ChangeType.UPSERT); - _entityClient.ingestProposal(proposal, context.getAuthentication()); + final MetadataChangeProposal proposal = buildMetadataChangeProposalWithUrn(entityUrn, OPERATION_ASPECT_NAME, + mapOperation(input, context)); + _entityClient.ingestProposal(proposal, context.getAuthentication(), false); return true; } catch (Exception e) { log.error("Failed to report operation. {}", e.getMessage()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/UpsertPolicyResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/UpsertPolicyResolver.java index f00e33361466f..6dcc143a1a3af 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/UpsertPolicyResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/UpsertPolicyResolver.java @@ -7,9 +7,7 @@ import com.linkedin.datahub.graphql.generated.PolicyUpdateInput; import com.linkedin.datahub.graphql.resolvers.policy.mappers.PolicyUpdateInputInfoMapper; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.key.DataHubPolicyKey; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.policy.DataHubPolicyInfo; import graphql.schema.DataFetcher; @@ -19,6 +17,8 @@ import java.util.concurrent.CompletableFuture; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; + public class UpsertPolicyResolver implements DataFetcher> { @@ -41,11 +41,14 @@ public CompletableFuture get(final DataFetchingEnvironment environment) final PolicyUpdateInput input = bindArgument(environment.getArgument("input"), PolicyUpdateInput.class); // Finally, create the MetadataChangeProposal. - final MetadataChangeProposal proposal = new MetadataChangeProposal(); + final MetadataChangeProposal proposal; + + final DataHubPolicyInfo info = PolicyUpdateInputInfoMapper.map(input); + info.setLastUpdatedTimestamp(System.currentTimeMillis()); if (policyUrn.isPresent()) { // Update existing policy - proposal.setEntityUrn(Urn.createFromString(policyUrn.get())); + proposal = buildMetadataChangeProposalWithUrn(Urn.createFromString(policyUrn.get()), POLICY_INFO_ASPECT_NAME, info); } else { // Create new policy // Since we are creating a new Policy, we need to generate a unique UUID. @@ -55,28 +58,18 @@ public CompletableFuture get(final DataFetchingEnvironment environment) // Create the Policy key. final DataHubPolicyKey key = new DataHubPolicyKey(); key.setId(uuidStr); - proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); + proposal = buildMetadataChangeProposalWithKey(key, POLICY_ENTITY_NAME, POLICY_INFO_ASPECT_NAME, info); } - // Create the policy info. - final DataHubPolicyInfo info = PolicyUpdateInputInfoMapper.map(input); - info.setLastUpdatedTimestamp(System.currentTimeMillis()); - - proposal.setEntityType(POLICY_ENTITY_NAME); - proposal.setAspectName(POLICY_INFO_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(info)); - proposal.setChangeType(ChangeType.UPSERT); - return CompletableFuture.supplyAsync(() -> { try { - // TODO: We should also provide SystemMetadata. - String urn = _entityClient.ingestProposal(proposal, context.getAuthentication()); + String urn = _entityClient.ingestProposal(proposal, context.getAuthentication(), false); if (context.getAuthorizer() instanceof AuthorizerChain) { ((AuthorizerChain) context.getAuthorizer()).getDefaultAuthorizer().invalidateCache(); } return urn; } catch (Exception e) { - throw new RuntimeException(String.format("Failed to perform update against input %s", input.toString()), e); + throw new RuntimeException(String.format("Failed to perform update against input %s", input), e); } }); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolver.java index a0568937d2651..9ee24e6941017 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolver.java @@ -10,12 +10,9 @@ import com.linkedin.datahub.graphql.generated.OwnershipType; import com.linkedin.datahub.graphql.resolvers.mutate.util.OwnerUtils; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.key.TagKey; import com.linkedin.metadata.utils.EntityKeyUtils; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.tag.TagProperties; import graphql.schema.DataFetcher; @@ -27,6 +24,8 @@ import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; import static com.linkedin.datahub.graphql.resolvers.mutate.util.OwnerUtils.*; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; +import static com.linkedin.metadata.Constants.*; /** @@ -59,19 +58,14 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws final String id = input.getId() != null ? input.getId() : UUID.randomUUID().toString(); key.setName(id); - if (_entityClient.exists(EntityKeyUtils.convertEntityKeyToUrn(key, Constants.TAG_ENTITY_NAME), context.getAuthentication())) { + if (_entityClient.exists(EntityKeyUtils.convertEntityKeyToUrn(key, TAG_ENTITY_NAME), context.getAuthentication())) { throw new IllegalArgumentException("This Tag already exists!"); } // Create the MCP - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); - proposal.setEntityType(Constants.TAG_ENTITY_NAME); - proposal.setAspectName(Constants.TAG_PROPERTIES_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(mapTagProperties(input))); - proposal.setChangeType(ChangeType.UPSERT); - - String tagUrn = _entityClient.ingestProposal(proposal, context.getAuthentication()); + final MetadataChangeProposal proposal = buildMetadataChangeProposalWithKey(key, TAG_ENTITY_NAME, + TAG_PROPERTIES_ASPECT_NAME, mapTagProperties(input)); + String tagUrn = _entityClient.ingestProposal(proposal, context.getAuthentication(), false); OwnershipType ownershipType = OwnershipType.TECHNICAL_OWNER; if (!_entityService.exists(UrnUtils.getUrn(mapOwnershipTypeToEntity(ownershipType.name())))) { log.warn("Technical owner does not exist, defaulting to None ownership."); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/SetTagColorResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/SetTagColorResolver.java index f6eff72ce51a0..b2781c5463b7f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/SetTagColorResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/SetTagColorResolver.java @@ -9,11 +9,8 @@ import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.resolvers.AuthUtils; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.tag.TagProperties; import graphql.schema.DataFetcher; @@ -24,6 +21,7 @@ import lombok.extern.slf4j.Slf4j; import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; +import static com.linkedin.metadata.Constants.*; /** @@ -59,7 +57,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw try { TagProperties tagProperties = (TagProperties) getAspectFromEntity( tagUrn.toString(), - Constants.TAG_PROPERTIES_ASPECT_NAME, + TAG_PROPERTIES_ASPECT_NAME, _entityService, null); @@ -70,13 +68,8 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw tagProperties.setColorHex(colorHex); // Update the TagProperties aspect. - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(tagUrn); - proposal.setEntityType(tagUrn.getEntityType()); - proposal.setAspectName(Constants.TAG_PROPERTIES_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(tagProperties)); - proposal.setChangeType(ChangeType.UPSERT); - _entityClient.ingestProposal(proposal, context.getAuthentication()); + final MetadataChangeProposal proposal = buildMetadataChangeProposalWithUrn(tagUrn, TAG_PROPERTIES_ASPECT_NAME, tagProperties); + _entityClient.ingestProposal(proposal, context.getAuthentication(), false); return true; } catch (Exception e) { log.error("Failed to set color for Tag with urn {}: {}", tagUrn, e.getMessage()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/test/CreateTestResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/test/CreateTestResolver.java index d792771df14d4..14ae9f96eb683 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/test/CreateTestResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/test/CreateTestResolver.java @@ -1,15 +1,13 @@ package com.linkedin.datahub.graphql.resolvers.test; +import com.datahub.authentication.Authentication; import com.linkedin.data.template.SetMode; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.generated.CreateTestInput; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.key.TestKey; import com.linkedin.metadata.utils.EntityKeyUtils; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.test.TestInfo; import graphql.schema.DataFetcher; @@ -18,7 +16,9 @@ import java.util.concurrent.CompletableFuture; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; import static com.linkedin.datahub.graphql.resolvers.test.TestUtils.*; +import static com.linkedin.metadata.Constants.*; /** @@ -35,6 +35,7 @@ public CreateTestResolver(final EntityClient entityClient) { @Override public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { final QueryContext context = environment.getContext(); + final Authentication authentication = context.getAuthentication(); final CreateTestInput input = bindArgument(environment.getArgument("input"), CreateTestInput.class); return CompletableFuture.supplyAsync(() -> { @@ -43,8 +44,6 @@ public CompletableFuture get(final DataFetchingEnvironment environment) try { - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - // Create new test // Since we are creating a new Test, we need to generate a unique UUID. final UUID uuid = UUID.randomUUID(); @@ -53,22 +52,19 @@ public CompletableFuture get(final DataFetchingEnvironment environment) // Create the Ingestion source key final TestKey key = new TestKey(); key.setId(uuidStr); - proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); - if (_entityClient.exists(EntityKeyUtils.convertEntityKeyToUrn(key, Constants.TEST_ENTITY_NAME), context.getAuthentication())) { + if (_entityClient.exists(EntityKeyUtils.convertEntityKeyToUrn(key, TEST_ENTITY_NAME), + authentication)) { throw new IllegalArgumentException("This Test already exists!"); } // Create the Test info. final TestInfo info = mapCreateTestInput(input); - proposal.setEntityType(Constants.TEST_ENTITY_NAME); - proposal.setAspectName(Constants.TEST_INFO_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(info)); - proposal.setChangeType(ChangeType.UPSERT); - return _entityClient.ingestProposal(proposal, context.getAuthentication()); + final MetadataChangeProposal proposal = buildMetadataChangeProposalWithKey(key, TEST_ENTITY_NAME, TEST_INFO_ASPECT_NAME, info); + return _entityClient.ingestProposal(proposal, context.getAuthentication(), false); } catch (Exception e) { - throw new RuntimeException(String.format("Failed to perform update against Test with urn %s", input.toString()), e); + throw new RuntimeException(String.format("Failed to perform update against Test with urn %s", input), e); } } throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/test/UpdateTestResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/test/UpdateTestResolver.java index bb37481ac8184..1dd8518076796 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/test/UpdateTestResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/test/UpdateTestResolver.java @@ -1,14 +1,12 @@ package com.linkedin.datahub.graphql.resolvers.test; +import com.datahub.authentication.Authentication; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.SetMode; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.generated.UpdateTestInput; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.Constants; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.test.TestInfo; import graphql.schema.DataFetcher; @@ -16,7 +14,9 @@ import java.util.concurrent.CompletableFuture; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; import static com.linkedin.datahub.graphql.resolvers.test.TestUtils.*; +import static com.linkedin.metadata.Constants.*; /** @@ -33,6 +33,7 @@ public UpdateTestResolver(final EntityClient entityClient) { @Override public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { final QueryContext context = environment.getContext(); + final Authentication authentication = context.getAuthentication(); return CompletableFuture.supplyAsync(() -> { @@ -40,20 +41,16 @@ public CompletableFuture get(final DataFetchingEnvironment environment) final String urn = environment.getArgument("urn"); final UpdateTestInput input = bindArgument(environment.getArgument("input"), UpdateTestInput.class); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); // Update the Test info - currently this simply creates a new test with same urn. final TestInfo info = mapUpdateTestInput(input); - proposal.setEntityUrn(UrnUtils.getUrn(urn)); - proposal.setEntityType(Constants.TEST_ENTITY_NAME); - proposal.setAspectName(Constants.TEST_INFO_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(info)); - proposal.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal = buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(urn), TEST_INFO_ASPECT_NAME, info); try { - return _entityClient.ingestProposal(proposal, context.getAuthentication()); + return _entityClient.ingestProposal(proposal, authentication, false); } catch (Exception e) { - throw new RuntimeException(String.format("Failed to perform update against Test with urn %s", input.toString()), e); + throw new RuntimeException( + String.format("Failed to perform update against Test with urn %s", input), e); } } throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/user/UpdateUserStatusResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/user/UpdateUserStatusResolver.java index e03c4ff968d4d..ab04d26fb5801 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/user/UpdateUserStatusResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/user/UpdateUserStatusResolver.java @@ -2,6 +2,7 @@ import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.linkedin.datahub.graphql.exception.AuthorizationException; @@ -12,6 +13,9 @@ import graphql.schema.DataFetchingEnvironment; import java.util.concurrent.CompletableFuture; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; +import static com.linkedin.metadata.Constants.*; + /** * Resolver responsible for editing a CorpUser's status. Requires the Manage Users & Groups platform privilege. @@ -39,9 +43,9 @@ public CompletableFuture get(final DataFetchingEnvironment environment) return CompletableFuture.supplyAsync(() -> { try { - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(userUrn)); - return _entityClient.ingestProposal(proposal, context.getAuthentication()); + final MetadataChangeProposal proposal = buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(userUrn), + CORP_USER_STATUS_ASPECT_NAME, statusAspect); + return _entityClient.ingestProposal(proposal, context.getAuthentication(), false); } catch (Exception e) { throw new RuntimeException(String.format("Failed to update user status for urn", userUrn), e); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/CorpGroupType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/CorpGroupType.java index 179c71aa76b6e..285a119be0d43 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/CorpGroupType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/CorpGroupType.java @@ -24,14 +24,12 @@ import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.identity.CorpGroupEditableInfo; import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.SearchFlags; import com.linkedin.metadata.search.SearchResult; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.execution.DataFetcherResult; import java.util.ArrayList; @@ -44,6 +42,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; import static com.linkedin.metadata.Constants.*; public class CorpGroupType implements SearchableEntityType, MutableType { @@ -137,14 +136,9 @@ public CorpGroup update(@Nonnull String urn, @Nonnull CorpGroupUpdateInput input } // Create the MCP - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(urn)); - proposal.setEntityType(CORP_GROUP_ENTITY_NAME); - proposal.setAspectName(CORP_GROUP_EDITABLE_INFO_ASPECT_NAME); - proposal.setAspect( - GenericRecordUtils.serializeAspect(mapCorpGroupEditableInfo(input, existingCorpGroupEditableInfo))); - proposal.setChangeType(ChangeType.UPSERT); - _entityClient.ingestProposal(proposal, context.getAuthentication()); + final MetadataChangeProposal proposal = buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(urn), + CORP_GROUP_EDITABLE_INFO_ASPECT_NAME, mapCorpGroupEditableInfo(input, existingCorpGroupEditableInfo)); + _entityClient.ingestProposal(proposal, context.getAuthentication(), false); return load(urn, context).getData(); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java index 1802e9087a725..db2b49c790f57 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java @@ -26,15 +26,12 @@ import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.identity.CorpUserEditableInfo; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.SearchFlags; import com.linkedin.metadata.search.SearchResult; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.execution.DataFetcherResult; import java.util.ArrayList; @@ -48,6 +45,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; import static com.linkedin.metadata.Constants.*; @@ -131,18 +129,13 @@ public CorpUser update(@Nonnull String urn, @Nonnull CorpUserUpdateInput input, if (isAuthorizedToUpdate(urn, input, context)) { // Get existing editable info to merge with Optional existingCorpUserEditableInfo = - _entityClient.getVersionedAspect(urn, Constants.CORP_USER_EDITABLE_INFO_NAME, 0L, CorpUserEditableInfo.class, + _entityClient.getVersionedAspect(urn, CORP_USER_EDITABLE_INFO_NAME, 0L, CorpUserEditableInfo.class, context.getAuthentication()); // Create the MCP - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(urn)); - proposal.setEntityType(Constants.CORP_USER_ENTITY_NAME); - proposal.setAspectName(Constants.CORP_USER_EDITABLE_INFO_NAME); - proposal.setAspect( - GenericRecordUtils.serializeAspect(mapCorpUserEditableInfo(input, existingCorpUserEditableInfo))); - proposal.setChangeType(ChangeType.UPSERT); - _entityClient.ingestProposal(proposal, context.getAuthentication()); + final MetadataChangeProposal proposal = buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(urn), + CORP_USER_EDITABLE_INFO_NAME, mapCorpUserEditableInfo(input, existingCorpUserEditableInfo)); + _entityClient.ingestProposal(proposal, context.getAuthentication(), false); return load(urn, context).getData(); } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/delete/BatchUpdateSoftDeletedResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/delete/BatchUpdateSoftDeletedResolverTest.java index 12cbf21b13666..7e549f201c2c7 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/delete/BatchUpdateSoftDeletedResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/delete/BatchUpdateSoftDeletedResolverTest.java @@ -8,10 +8,9 @@ import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.BatchUpdateSoftDeletedInput; import com.linkedin.datahub.graphql.resolvers.mutate.BatchUpdateSoftDeletedResolver; -import com.linkedin.events.metadata.ChangeType; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetchingEnvironment; import java.util.concurrent.CompletionException; @@ -19,6 +18,7 @@ import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.metadata.Constants.*; import static org.testng.Assert.*; @@ -59,21 +59,13 @@ public void testGetSuccessNoExistingStatus() throws Exception { final Status newStatus = new Status().setRemoved(true); - final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); - proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); - proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal1.setAspectName(Constants.STATUS_ASPECT_NAME); - proposal1.setAspect(GenericRecordUtils.serializeAspect(newStatus)); - proposal1.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal1 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_1), + STATUS_ASPECT_NAME, newStatus); verifyIngestProposal(mockService, 1, proposal1); - final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); - proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); - proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal2.setAspectName(Constants.STATUS_ASPECT_NAME); - proposal2.setAspect(GenericRecordUtils.serializeAspect(newStatus)); - proposal2.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal2 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_2), + STATUS_ASPECT_NAME, newStatus); verifyIngestProposal(mockService, 1, proposal2); } @@ -111,21 +103,13 @@ public void testGetSuccessExistingStatus() throws Exception { final Status newStatus = new Status().setRemoved(false); - final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); - proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); - proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal1.setAspectName(Constants.STATUS_ASPECT_NAME); - proposal1.setAspect(GenericRecordUtils.serializeAspect(newStatus)); - proposal1.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal1 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_1), + STATUS_ASPECT_NAME, newStatus); verifyIngestProposal(mockService, 1, proposal1); - final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); - proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); - proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal2.setAspectName(Constants.STATUS_ASPECT_NAME); - proposal2.setAspect(GenericRecordUtils.serializeAspect(newStatus)); - proposal2.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal2 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_2), + STATUS_ASPECT_NAME, newStatus); verifyIngestProposal(mockService, 1, proposal2); } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/deprecation/BatchUpdateDeprecationResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/deprecation/BatchUpdateDeprecationResolverTest.java index 36909eb075d9b..634fd59a857a6 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/deprecation/BatchUpdateDeprecationResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/deprecation/BatchUpdateDeprecationResolverTest.java @@ -9,10 +9,9 @@ import com.linkedin.datahub.graphql.generated.BatchUpdateDeprecationInput; import com.linkedin.datahub.graphql.generated.ResourceRefInput; import com.linkedin.datahub.graphql.resolvers.mutate.BatchUpdateDeprecationResolver; -import com.linkedin.events.metadata.ChangeType; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetchingEnvironment; import java.util.concurrent.CompletionException; @@ -20,6 +19,7 @@ import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.metadata.Constants.*; import static org.testng.Assert.*; @@ -66,22 +66,13 @@ public void testGetSuccessNoExistingDeprecation() throws Exception { .setDecommissionTime(0L) .setActor(UrnUtils.getUrn("urn:li:corpuser:test")); - final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); - proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); - proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal1.setAspectName(Constants.DEPRECATION_ASPECT_NAME); - proposal1.setAspect(GenericRecordUtils.serializeAspect(newDeprecation)); - proposal1.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal1 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_1), + DEPRECATION_ASPECT_NAME, newDeprecation); verifyIngestProposal(mockService, 1, proposal1); - final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); - proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); - proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal2.setAspectName(Constants.DEPRECATION_ASPECT_NAME); - proposal2.setAspect(GenericRecordUtils.serializeAspect(newDeprecation)); - proposal2.setChangeType(ChangeType.UPSERT); - + final MetadataChangeProposal proposal2 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_2), + DEPRECATION_ASPECT_NAME, newDeprecation); verifyIngestProposal(mockService, 1, proposal2); } @@ -127,21 +118,13 @@ public void testGetSuccessExistingDeprecation() throws Exception { .setDecommissionTime(1L) .setActor(UrnUtils.getUrn("urn:li:corpuser:test")); - final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); - proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); - proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal1.setAspectName(Constants.DEPRECATION_ASPECT_NAME); - proposal1.setAspect(GenericRecordUtils.serializeAspect(newDeprecation)); - proposal1.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal1 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_1), + DEPRECATION_ASPECT_NAME, newDeprecation); verifyIngestProposal(mockService, 1, proposal1); - final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); - proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); - proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal2.setAspectName(Constants.DEPRECATION_ASPECT_NAME); - proposal2.setAspect(GenericRecordUtils.serializeAspect(newDeprecation)); - proposal2.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal2 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_2), + DEPRECATION_ASPECT_NAME, newDeprecation); verifyIngestProposal(mockService, 1, proposal2); } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/deprecation/UpdateDeprecationResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/deprecation/UpdateDeprecationResolverTest.java index a41d1be2a00b3..cf11ecf865085 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/deprecation/UpdateDeprecationResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/deprecation/UpdateDeprecationResolverTest.java @@ -6,17 +6,17 @@ import com.linkedin.common.Deprecation; import com.linkedin.common.urn.CorpuserUrn; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.UpdateDeprecationInput; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.entity.Aspect; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetchingEnvironment; @@ -27,6 +27,7 @@ import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.metadata.Constants.*; import static org.testng.Assert.*; @@ -71,16 +72,13 @@ public void testGetSuccessNoExistingDeprecation() throws Exception { resolver.get(mockEnv).get(); final Deprecation newDeprecation = new Deprecation().setDeprecated(true).setDecommissionTime(0L).setNote("Test note").setActor(TEST_ACTOR_URN); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN)); - proposal.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal.setAspectName(Constants.DEPRECATION_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(newDeprecation)); - proposal.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal = MutationUtils.buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(TEST_ENTITY_URN), + DEPRECATION_ASPECT_NAME, newDeprecation); Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( Mockito.eq(proposal), - Mockito.any(Authentication.class) + Mockito.any(Authentication.class), + Mockito.eq(false) ); Mockito.verify(mockService, Mockito.times(1)).exists( @@ -127,16 +125,13 @@ public void testGetSuccessExistingDeprecation() throws Exception { .setDecommissionTime(0L) .setNote("Test note") .setActor(TEST_ACTOR_URN); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN)); - proposal.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal.setAspectName(Constants.DEPRECATION_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(newDeprecation)); - proposal.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal = MutationUtils.buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(TEST_ENTITY_URN), + DEPRECATION_ASPECT_NAME, newDeprecation); Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( Mockito.eq(proposal), - Mockito.any(Authentication.class) + Mockito.any(Authentication.class), + Mockito.eq(false) ); Mockito.verify(mockService, Mockito.times(1)).exists( diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/BatchSetDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/BatchSetDomainResolverTest.java index fe3bfb3dec66b..9241661ccfc87 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/BatchSetDomainResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/BatchSetDomainResolverTest.java @@ -9,6 +9,7 @@ import com.linkedin.datahub.graphql.generated.BatchSetDomainInput; import com.linkedin.datahub.graphql.generated.ResourceRefInput; import com.linkedin.datahub.graphql.resolvers.mutate.BatchSetDomainResolver; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.domain.Domains; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; @@ -21,6 +22,7 @@ import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.metadata.Constants.*; import static org.testng.Assert.*; @@ -70,21 +72,13 @@ public void testGetSuccessNoExistingDomains() throws Exception { Urn.createFromString(TEST_DOMAIN_2_URN) ))); - final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); - proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); - proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal1.setAspectName(Constants.DOMAINS_ASPECT_NAME); - proposal1.setAspect(GenericRecordUtils.serializeAspect(newDomains)); - proposal1.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal1 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_1), + DOMAINS_ASPECT_NAME, newDomains); verifyIngestProposal(mockService, 1, proposal1); - final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); - proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); - proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal2.setAspectName(Constants.DOMAINS_ASPECT_NAME); - proposal2.setAspect(GenericRecordUtils.serializeAspect(newDomains)); - proposal2.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal2 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_2), + DOMAINS_ASPECT_NAME, newDomains); verifyIngestProposal(mockService, 1, proposal2); @@ -134,7 +128,8 @@ public void testGetSuccessExistingDomains() throws Exception { Urn.createFromString(TEST_DOMAIN_2_URN) ))); - final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); + final MetadataChangeProposal proposal1 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_1), + DOMAINS_ASPECT_NAME, newDomains); proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); proposal1.setAspectName(Constants.DOMAINS_ASPECT_NAME); @@ -143,12 +138,8 @@ public void testGetSuccessExistingDomains() throws Exception { verifyIngestProposal(mockService, 1, proposal1); - final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); - proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); - proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal2.setAspectName(Constants.DOMAINS_ASPECT_NAME); - proposal2.setAspect(GenericRecordUtils.serializeAspect(newDomains)); - proposal2.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal2 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_2), + DOMAINS_ASPECT_NAME, newDomains); verifyIngestProposal(mockService, 1, proposal2); @@ -196,21 +187,13 @@ public void testGetSuccessUnsetDomains() throws Exception { final Domains newDomains = new Domains().setDomains(new UrnArray(ImmutableList.of())); - final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); - proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); - proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal1.setAspectName(Constants.DOMAINS_ASPECT_NAME); - proposal1.setAspect(GenericRecordUtils.serializeAspect(newDomains)); - proposal1.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal1 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_1), + DOMAINS_ASPECT_NAME, newDomains); verifyIngestProposal(mockService, 1, proposal1); - final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); - proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); - proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal2.setAspectName(Constants.DOMAINS_ASPECT_NAME); - proposal2.setAspect(GenericRecordUtils.serializeAspect(newDomains)); - proposal2.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal2 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_1), + DOMAINS_ASPECT_NAME, newDomains); verifyIngestProposal(mockService, 1, proposal2); } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java index 7e8b01950332d..9343e5d772826 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java @@ -67,7 +67,8 @@ public void testGetSuccess() throws Exception { // Not ideal to match against "any", but we don't know the auto-generated execution request id Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( Mockito.argThat(new CreateDomainProposalMatcher(proposal)), - Mockito.any(Authentication.class) + Mockito.any(Authentication.class), + Mockito.eq(false) ); } @@ -97,7 +98,7 @@ public void testGetEntityClientException() throws Exception { EntityService mockService = Mockito.mock(EntityService.class); Mockito.doThrow(RemoteInvocationException.class).when(mockClient).ingestProposal( Mockito.any(), - Mockito.any(Authentication.class)); + Mockito.any(Authentication.class), Mockito.eq(false)); CreateDomainResolver resolver = new CreateDomainResolver(mockClient, mockService); // Execute resolver diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/SetDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/SetDomainResolverTest.java index c25cb984b7c60..73d1f699dfd80 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/SetDomainResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/SetDomainResolverTest.java @@ -6,17 +6,17 @@ import com.google.common.collect.ImmutableSet; import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.domain.Domains; import com.linkedin.entity.Aspect; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetchingEnvironment; @@ -27,6 +27,7 @@ import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.metadata.Constants.*; import static org.testng.Assert.*; @@ -68,16 +69,13 @@ public void testGetSuccessNoExistingDomains() throws Exception { resolver.get(mockEnv).get(); final Domains newDomains = new Domains().setDomains(new UrnArray(ImmutableList.of(Urn.createFromString(TEST_NEW_DOMAIN_URN)))); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN)); - proposal.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal.setAspectName(Constants.DOMAINS_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(newDomains)); - proposal.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal = MutationUtils.buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(TEST_ENTITY_URN), + DOMAINS_ASPECT_NAME, newDomains); Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( Mockito.eq(proposal), - Mockito.any(Authentication.class) + Mockito.any(Authentication.class), + Mockito.eq(false) ); Mockito.verify(mockService, Mockito.times(1)).exists( @@ -127,16 +125,13 @@ public void testGetSuccessExistingDomains() throws Exception { resolver.get(mockEnv).get(); final Domains newDomains = new Domains().setDomains(new UrnArray(ImmutableList.of(Urn.createFromString(TEST_NEW_DOMAIN_URN)))); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN)); - proposal.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal.setAspectName(Constants.DOMAINS_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(newDomains)); - proposal.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal = MutationUtils.buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(TEST_ENTITY_URN), + DOMAINS_ASPECT_NAME, newDomains); Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( Mockito.eq(proposal), - Mockito.any(Authentication.class) + Mockito.any(Authentication.class), + Mockito.eq(false) ); Mockito.verify(mockService, Mockito.times(1)).exists( diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/UnsetDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/UnsetDomainResolverTest.java index be5343f6d2172..18b2b9a2747e2 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/UnsetDomainResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/UnsetDomainResolverTest.java @@ -6,17 +6,17 @@ import com.google.common.collect.ImmutableSet; import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.domain.Domains; import com.linkedin.entity.Aspect; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetchingEnvironment; @@ -27,6 +27,7 @@ import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.metadata.Constants.*; import static org.testng.Assert.*; @@ -65,16 +66,13 @@ public void testGetSuccessNoExistingDomains() throws Exception { resolver.get(mockEnv).get(); final Domains newDomains = new Domains().setDomains(new UrnArray(Collections.emptyList())); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN)); - proposal.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal.setAspectName(Constants.DOMAINS_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(newDomains)); - proposal.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal = MutationUtils.buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(TEST_ENTITY_URN), + DOMAINS_ASPECT_NAME, newDomains); Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( Mockito.eq(proposal), - Mockito.any(Authentication.class) + Mockito.any(Authentication.class), + Mockito.eq(false) ); Mockito.verify(mockService, Mockito.times(1)).exists( @@ -119,16 +117,13 @@ public void testGetSuccessExistingDomains() throws Exception { resolver.get(mockEnv).get(); final Domains newDomains = new Domains().setDomains(new UrnArray(Collections.emptyList())); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN)); - proposal.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal.setAspectName(Constants.DOMAINS_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(newDomains)); - proposal.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal = MutationUtils.buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(TEST_ENTITY_URN), + DOMAINS_ASPECT_NAME, newDomains); Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( Mockito.eq(proposal), - Mockito.any(Authentication.class) + Mockito.any(Authentication.class), + Mockito.eq(false) ); Mockito.verify(mockService, Mockito.times(1)).exists( diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/embed/UpdateEmbedResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/embed/UpdateEmbedResolverTest.java index 94dc742d77831..17f7a1968fdf6 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/embed/UpdateEmbedResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/embed/UpdateEmbedResolverTest.java @@ -9,13 +9,12 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.UpdateEmbedInput; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetchingEnvironment; @@ -26,6 +25,7 @@ import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.metadata.Constants.*; import static org.testng.Assert.*; @@ -45,7 +45,7 @@ public void testGetSuccessNoExistingEmbed() throws Exception { Mockito.when(mockService.getAspect( Mockito.eq(Urn.createFromString(TEST_ENTITY_URN)), - Mockito.eq(Constants.EMBED_ASPECT_NAME), + Mockito.eq(EMBED_ASPECT_NAME), Mockito.eq(0L))).thenReturn(null); Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN))).thenReturn(true); @@ -61,12 +61,8 @@ public void testGetSuccessNoExistingEmbed() throws Exception { resolver.get(mockEnv).get(); final Embed newEmbed = new Embed().setRenderUrl(TEST_RENDER_URL); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN)); - proposal.setEntityType(Constants.DASHBOARD_ENTITY_NAME); - proposal.setAspectName(Constants.EMBED_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(newEmbed)); - proposal.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN), + EMBED_ASPECT_NAME, newEmbed); Mockito.verify(mockService, Mockito.times(1)).ingestProposal( Mockito.eq(proposal), @@ -88,7 +84,7 @@ public void testGetSuccessExistingEmbed() throws Exception { Mockito.when(mockService.getAspect( Mockito.eq(Urn.createFromString(TEST_ENTITY_URN)), - Mockito.eq(Constants.EMBED_ASPECT_NAME), + Mockito.eq(EMBED_ASPECT_NAME), Mockito.eq(0L))).thenReturn(originalEmbed); Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN))).thenReturn(true); @@ -104,12 +100,8 @@ public void testGetSuccessExistingEmbed() throws Exception { resolver.get(mockEnv).get(); final Embed newEmbed = new Embed().setRenderUrl(TEST_RENDER_URL); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN)); - proposal.setEntityType(Constants.DASHBOARD_ENTITY_NAME); - proposal.setAspectName(Constants.EMBED_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(newEmbed)); - proposal.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN), + EMBED_ASPECT_NAME, newEmbed); Mockito.verify(mockService, Mockito.times(1)).ingestProposal( Mockito.eq(proposal), @@ -130,7 +122,7 @@ public void testGetFailureEntityDoesNotExist() throws Exception { Mockito.when(mockClient.batchGetV2( Mockito.eq(Constants.DASHBOARD_ENTITY_NAME), Mockito.eq(new HashSet<>(ImmutableSet.of(Urn.createFromString(TEST_ENTITY_URN)))), - Mockito.eq(ImmutableSet.of(Constants.EMBED_ASPECT_NAME)), + Mockito.eq(ImmutableSet.of(EMBED_ASPECT_NAME)), Mockito.any(Authentication.class))) .thenReturn(ImmutableMap.of(Urn.createFromString(TEST_ENTITY_URN), new EntityResponse() diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolverTest.java index 7ee810f3c0454..392ddf6ac4c74 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolverTest.java @@ -4,12 +4,10 @@ import com.linkedin.common.urn.GlossaryNodeUrn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.CreateGlossaryEntityInput; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.glossary.GlossaryNodeInfo; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.key.GlossaryNodeKey; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.metadata.entity.EntityService; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetchingEnvironment; @@ -17,6 +15,8 @@ import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.metadata.Constants.*; + public class CreateGlossaryNodeResolverTest { @@ -55,9 +55,6 @@ private MetadataChangeProposal setupTest( final GlossaryNodeKey key = new GlossaryNodeKey(); key.setName("test-id"); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); - proposal.setEntityType(Constants.GLOSSARY_NODE_ENTITY_NAME); GlossaryNodeInfo props = new GlossaryNodeInfo(); props.setDefinition(description); props.setName("test-name"); @@ -65,11 +62,8 @@ private MetadataChangeProposal setupTest( final GlossaryNodeUrn parent = GlossaryNodeUrn.createFromString(parentNode); props.setParentNode(parent); } - proposal.setAspectName(Constants.GLOSSARY_NODE_INFO_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(props)); - proposal.setChangeType(ChangeType.UPSERT); - - return proposal; + return MutationUtils.buildMetadataChangeProposalWithKey(key, GLOSSARY_NODE_ENTITY_NAME, + GLOSSARY_NODE_INFO_ASPECT_NAME, props); } @Test @@ -84,7 +78,8 @@ public void testGetSuccess() throws Exception { Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( Mockito.eq(proposal), - Mockito.any(Authentication.class) + Mockito.any(Authentication.class), + Mockito.eq(false) ); } @@ -100,7 +95,8 @@ public void testGetSuccessNoDescription() throws Exception { Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( Mockito.eq(proposal), - Mockito.any(Authentication.class) + Mockito.any(Authentication.class), + Mockito.eq(false) ); } @@ -116,7 +112,8 @@ public void testGetSuccessNoParentNode() throws Exception { Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( Mockito.eq(proposal), - Mockito.any(Authentication.class) + Mockito.any(Authentication.class), + Mockito.eq(false) ); } } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryTermResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryTermResolverTest.java index a8b640898929f..929162db1cc43 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryTermResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryTermResolverTest.java @@ -4,12 +4,10 @@ import com.linkedin.common.urn.GlossaryNodeUrn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.CreateGlossaryEntityInput; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.glossary.GlossaryTermInfo; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.key.GlossaryTermKey; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.metadata.entity.EntityService; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetchingEnvironment; @@ -17,6 +15,8 @@ import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.metadata.Constants.*; + public class CreateGlossaryTermResolverTest { @@ -55,9 +55,6 @@ private MetadataChangeProposal setupTest( final GlossaryTermKey key = new GlossaryTermKey(); key.setName("test-id"); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); - proposal.setEntityType(Constants.GLOSSARY_TERM_ENTITY_NAME); GlossaryTermInfo props = new GlossaryTermInfo(); props.setDefinition(description); props.setName("test-name"); @@ -66,11 +63,8 @@ private MetadataChangeProposal setupTest( final GlossaryNodeUrn parent = GlossaryNodeUrn.createFromString(parentNode); props.setParentNode(parent); } - proposal.setAspectName(Constants.GLOSSARY_TERM_INFO_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(props)); - proposal.setChangeType(ChangeType.UPSERT); - - return proposal; + return MutationUtils.buildMetadataChangeProposalWithKey(key, GLOSSARY_TERM_ENTITY_NAME, + GLOSSARY_TERM_INFO_ASPECT_NAME, props); } @Test @@ -85,7 +79,8 @@ public void testGetSuccess() throws Exception { Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( Mockito.eq(proposal), - Mockito.any(Authentication.class) + Mockito.any(Authentication.class), + Mockito.eq(false) ); } @@ -101,7 +96,8 @@ public void testGetSuccessNoDescription() throws Exception { Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( Mockito.eq(proposal), - Mockito.any(Authentication.class) + Mockito.any(Authentication.class), + Mockito.eq(false) ); } @@ -117,7 +113,8 @@ public void testGetSuccessNoParentNode() throws Exception { Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( Mockito.eq(proposal), - Mockito.any(Authentication.class) + Mockito.any(Authentication.class), + Mockito.eq(false) ); } } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateNameResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateNameResolverTest.java index 9e96e27abf635..36f909bd7ebe5 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateNameResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateNameResolverTest.java @@ -6,15 +6,14 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.UpdateNameInput; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.datahub.graphql.resolvers.mutate.UpdateNameResolver; import com.linkedin.domain.DomainProperties; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.glossary.GlossaryNodeInfo; import com.linkedin.glossary.GlossaryTermInfo; import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetchingEnvironment; import org.mockito.Mockito; @@ -23,6 +22,7 @@ import java.util.concurrent.CompletionException; import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.metadata.Constants.*; import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; @@ -50,15 +50,10 @@ private MetadataChangeProposal setupTests(DataFetchingEnvironment mockEnv, Entit 0)) .thenReturn(new GlossaryTermInfo().setName(name)); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(TERM_URN)); - proposal.setEntityType(Constants.GLOSSARY_TERM_ENTITY_NAME); GlossaryTermInfo info = new GlossaryTermInfo(); info.setName(NEW_NAME); - proposal.setAspectName(Constants.GLOSSARY_TERM_INFO_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(info)); - proposal.setChangeType(ChangeType.UPSERT); - return proposal; + return MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TERM_URN), + GLOSSARY_TERM_INFO_ASPECT_NAME, info); } @Test @@ -96,15 +91,10 @@ public void testGetSuccessForNode() throws Exception { 0)) .thenReturn(new GlossaryNodeInfo().setName(name)); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(NODE_URN)); - proposal.setEntityType(Constants.GLOSSARY_NODE_ENTITY_NAME); GlossaryNodeInfo info = new GlossaryNodeInfo(); info.setName(NEW_NAME); - proposal.setAspectName(Constants.GLOSSARY_NODE_INFO_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(info)); - proposal.setChangeType(ChangeType.UPSERT); - + final MetadataChangeProposal proposal = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(NODE_URN), + GLOSSARY_NODE_INFO_ASPECT_NAME, info); UpdateNameResolver resolver = new UpdateNameResolver(mockService, mockClient); assertTrue(resolver.get(mockEnv).get()); @@ -131,14 +121,10 @@ public void testGetSuccessForDomain() throws Exception { 0)) .thenReturn(new DomainProperties().setName(name)); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(DOMAIN_URN)); - proposal.setEntityType(Constants.DOMAIN_ENTITY_NAME); DomainProperties properties = new DomainProperties(); properties.setName(NEW_NAME); - proposal.setAspectName(Constants.DOMAIN_PROPERTIES_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(properties)); - proposal.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(DOMAIN_URN), + DOMAIN_PROPERTIES_ASPECT_NAME, properties); UpdateNameResolver resolver = new UpdateNameResolver(mockService, mockClient); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateParentNodeResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateParentNodeResolverTest.java index 1cbe5484e7b8a..43c6113d194a5 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateParentNodeResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateParentNodeResolverTest.java @@ -6,14 +6,13 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.UpdateParentNodeInput; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.datahub.graphql.resolvers.mutate.UpdateParentNodeResolver; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.glossary.GlossaryNodeInfo; import com.linkedin.glossary.GlossaryTermInfo; import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetchingEnvironment; import org.mockito.Mockito; @@ -22,6 +21,7 @@ import java.net.URISyntaxException; import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.metadata.Constants.*; import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; @@ -49,16 +49,11 @@ private MetadataChangeProposal setupTests(DataFetchingEnvironment mockEnv, Entit 0)) .thenReturn(new GlossaryTermInfo().setName(name)); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(TERM_URN)); - proposal.setEntityType(Constants.GLOSSARY_TERM_ENTITY_NAME); GlossaryTermInfo info = new GlossaryTermInfo(); info.setName(name); info.setParentNode(GlossaryNodeUrn.createFromString(PARENT_NODE_URN)); - proposal.setAspectName(Constants.GLOSSARY_TERM_INFO_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(info)); - proposal.setChangeType(ChangeType.UPSERT); - return proposal; + return MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TERM_URN), + GLOSSARY_TERM_INFO_ASPECT_NAME, info); } @Test @@ -98,15 +93,11 @@ public void testGetSuccessForNode() throws Exception { 0)) .thenReturn(new GlossaryNodeInfo().setName(name)); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(NODE_URN)); - proposal.setEntityType(Constants.GLOSSARY_NODE_ENTITY_NAME); GlossaryNodeInfo info = new GlossaryNodeInfo(); info.setName(name); info.setParentNode(GlossaryNodeUrn.createFromString(PARENT_NODE_URN)); - proposal.setAspectName(Constants.GLOSSARY_NODE_INFO_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(info)); - proposal.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(NODE_URN), + GLOSSARY_NODE_INFO_ASPECT_NAME, info); UpdateParentNodeResolver resolver = new UpdateParentNodeResolver(mockService, mockClient); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CancelIngestionExecutionRequestResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CancelIngestionExecutionRequestResolverTest.java index 9c58faf1b67e7..e7226c6e4db08 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CancelIngestionExecutionRequestResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CancelIngestionExecutionRequestResolverTest.java @@ -59,7 +59,8 @@ Constants.INGESTION_INFO_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect( // Verify ingest proposal has been called to create a Signal request. Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( Mockito.any(MetadataChangeProposal.class), - Mockito.any(Authentication.class) + Mockito.any(Authentication.class), + Mockito.eq(false) ); } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolverTest.java index 55a8c40f15766..35043fa0879f3 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolverTest.java @@ -60,7 +60,8 @@ public void testGetSuccess() throws Exception { // Not ideal to match against "any", but we don't know the auto-generated execution request id Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( Mockito.any(MetadataChangeProposal.class), - Mockito.any(Authentication.class) + Mockito.any(Authentication.class), + Mockito.eq(false) ); } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolverTest.java index bda576b7afd8e..0eb9366f0493b 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolverTest.java @@ -39,7 +39,8 @@ public void testGetSuccess() throws Exception { Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( Mockito.any(MetadataChangeProposal.class), - Mockito.any(Authentication.class) + Mockito.any(Authentication.class), + Mockito.eq(false) ); } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/CreateSecretResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/CreateSecretResolverTest.java index 817ea62af3cac..18ae71661318e 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/CreateSecretResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/CreateSecretResolverTest.java @@ -66,7 +66,8 @@ public void testGetSuccess() throws Exception { .setAspectName(Constants.SECRET_VALUE_ASPECT_NAME) .setAspect(GenericRecordUtils.serializeAspect(value)) .setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)))), - Mockito.any(Authentication.class) + Mockito.any(Authentication.class), + Mockito.eq(false) ); } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolverTest.java index 99f97bc73f577..2538accc694fb 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolverTest.java @@ -5,20 +5,18 @@ import com.linkedin.datahub.graphql.generated.UpdateIngestionSourceConfigInput; import com.linkedin.datahub.graphql.generated.UpdateIngestionSourceInput; import com.linkedin.datahub.graphql.generated.UpdateIngestionSourceScheduleInput; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.ingestion.DataHubIngestionSourceConfig; import com.linkedin.ingestion.DataHubIngestionSourceInfo; import com.linkedin.ingestion.DataHubIngestionSourceSchedule; -import com.linkedin.metadata.Constants; -import com.linkedin.metadata.utils.GenericRecordUtils; -import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetchingEnvironment; import org.mockito.Mockito; import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.resolvers.ingest.IngestTestUtils.*; +import static com.linkedin.metadata.Constants.*; import static org.testng.Assert.*; @@ -62,15 +60,11 @@ public void testGetSuccess() throws Exception { ); Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( - Mockito.eq( - new MetadataChangeProposal() - .setChangeType(ChangeType.UPSERT) - .setEntityType(Constants.INGESTION_SOURCE_ENTITY_NAME) - .setAspectName(Constants.INGESTION_INFO_ASPECT_NAME) - .setAspect(GenericRecordUtils.serializeAspect(info)) - .setEntityUrn(TEST_INGESTION_SOURCE_URN) + Mockito.eq(MutationUtils.buildMetadataChangeProposalWithUrn(TEST_INGESTION_SOURCE_URN, + INGESTION_INFO_ASPECT_NAME, info) ), - Mockito.any(Authentication.class) + Mockito.any(Authentication.class), + Mockito.eq(false) ); } @@ -99,7 +93,8 @@ public void testGetEntityClientException() throws Exception { EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.doThrow(RemoteInvocationException.class).when(mockClient).ingestProposal( Mockito.any(), - Mockito.any(Authentication.class)); + Mockito.any(Authentication.class), + Mockito.eq(false)); UpsertIngestionSourceResolver resolver = new UpsertIngestionSourceResolver(mockClient); // Execute resolver diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateUserSettingResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateUserSettingResolverTest.java index 605f1e4142e18..c7f1e16a0ea61 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateUserSettingResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateUserSettingResolverTest.java @@ -4,18 +4,16 @@ import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.UpdateUserSettingInput; import com.linkedin.datahub.graphql.generated.UserSetting; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.identity.CorpUserAppearanceSettings; import com.linkedin.identity.CorpUserSettings; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetchingEnvironment; import org.mockito.Mockito; import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.metadata.Constants.*; public class UpdateUserSettingResolverTest { @@ -39,12 +37,8 @@ public void testWriteCorpUserSettings() throws Exception { resolver.get(mockEnv).get(); CorpUserSettings newSettings = new CorpUserSettings().setAppearance(new CorpUserAppearanceSettings().setShowSimplifiedHomepage(true)); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(TEST_USER_URN)); - proposal.setEntityType(Constants.CORP_USER_ENTITY_NAME); - proposal.setAspectName(Constants.CORP_USER_SETTINGS_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(newSettings)); - proposal.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_USER_URN), + CORP_USER_SETTINGS_ASPECT_NAME, newSettings); verifyIngestProposal(mockService, 1, proposal); } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/operation/ReportOperationResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/operation/ReportOperationResolverTest.java index 46035c0f7bbe5..e2661841fe8f7 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/operation/ReportOperationResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/operation/ReportOperationResolverTest.java @@ -8,10 +8,8 @@ import com.linkedin.data.template.SetMode; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.ReportOperationInput; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.Constants; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetchingEnvironment; import java.util.concurrent.CompletionException; @@ -19,6 +17,7 @@ import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.metadata.Constants.*; import static org.testng.Assert.*; @@ -40,12 +39,8 @@ public void testGetSuccess() throws Exception { .setCustomOperationType(null, SetMode.IGNORE_NULL) .setNumAffectedRows(1L); - MetadataChangeProposal expectedProposal = new MetadataChangeProposal() - .setAspectName(Constants.OPERATION_ASPECT_NAME) - .setChangeType(ChangeType.UPSERT) - .setEntityUrn(UrnUtils.getUrn(TEST_ENTITY_URN)) - .setEntityType(Constants.DATASET_ENTITY_NAME) - .setAspect(GenericRecordUtils.serializeAspect(expectedOperation)); + MetadataChangeProposal expectedProposal = MutationUtils.buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(TEST_ENTITY_URN), + OPERATION_ASPECT_NAME, expectedOperation); // Test setting the domain Mockito.when(mockClient.ingestProposal( @@ -64,7 +59,8 @@ public void testGetSuccess() throws Exception { Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( Mockito.eq(expectedProposal), - Mockito.any(Authentication.class) + Mockito.any(Authentication.class), + Mockito.eq(false) ); } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/AddTagsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/AddTagsResolverTest.java index e0769668be023..06d9df3278847 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/AddTagsResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/AddTagsResolverTest.java @@ -11,10 +11,8 @@ import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.AddTagsInput; import com.linkedin.datahub.graphql.resolvers.mutate.AddTagsResolver; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.Constants; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetchingEnvironment; import java.util.concurrent.CompletionException; @@ -22,6 +20,7 @@ import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.metadata.Constants.*; import static org.testng.Assert.*; @@ -37,7 +36,7 @@ public void testGetSuccessNoExistingTags() throws Exception { Mockito.when(mockService.getAspect( Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN)), - Mockito.eq(Constants.GLOBAL_TAGS_ASPECT_NAME), + Mockito.eq(GLOBAL_TAGS_ASPECT_NAME), Mockito.eq(0L))) .thenReturn(null); @@ -63,12 +62,8 @@ public void testGetSuccessNoExistingTags() throws Exception { new TagAssociation().setTag(TagUrn.createFromString(TEST_TAG_2_URN)) ))); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN)); - proposal.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal.setAspectName(Constants.GLOBAL_TAGS_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(newTags)); - proposal.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN), + GLOBAL_TAGS_ASPECT_NAME, newTags); verifyIngestProposal(mockService, 1, proposal); @@ -91,7 +86,7 @@ public void testGetSuccessExistingTags() throws Exception { Mockito.when(mockService.getAspect( Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN)), - Mockito.eq(Constants.GLOBAL_TAGS_ASPECT_NAME), + Mockito.eq(GLOBAL_TAGS_ASPECT_NAME), Mockito.eq(0L))) .thenReturn(originalTags); @@ -117,12 +112,8 @@ public void testGetSuccessExistingTags() throws Exception { new TagAssociation().setTag(TagUrn.createFromString(TEST_TAG_2_URN)) ))); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN)); - proposal.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal.setAspectName(Constants.GLOBAL_TAGS_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(newTags)); - proposal.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN), + GLOBAL_TAGS_ASPECT_NAME, newTags); verifyIngestProposal(mockService, 1, proposal); @@ -141,7 +132,7 @@ public void testGetFailureTagDoesNotExist() throws Exception { Mockito.when(mockService.getAspect( Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN)), - Mockito.eq(Constants.GLOBAL_TAGS_ASPECT_NAME), + Mockito.eq(GLOBAL_TAGS_ASPECT_NAME), Mockito.eq(0L))) .thenReturn(null); @@ -169,7 +160,7 @@ public void testGetFailureResourceDoesNotExist() throws Exception { Mockito.when(mockService.getAspect( Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN)), - Mockito.eq(Constants.GLOBAL_TAGS_ASPECT_NAME), + Mockito.eq(GLOBAL_TAGS_ASPECT_NAME), Mockito.eq(0L))) .thenReturn(null); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchAddTagsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchAddTagsResolverTest.java index 4991f1b59a0f9..d47e6164fe221 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchAddTagsResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchAddTagsResolverTest.java @@ -12,10 +12,9 @@ import com.linkedin.datahub.graphql.generated.BatchAddTagsInput; import com.linkedin.datahub.graphql.generated.ResourceRefInput; import com.linkedin.datahub.graphql.resolvers.mutate.BatchAddTagsResolver; -import com.linkedin.events.metadata.ChangeType; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetchingEnvironment; import java.util.concurrent.CompletionException; @@ -23,6 +22,7 @@ import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.metadata.Constants.*; import static org.testng.Assert.*; @@ -76,21 +76,13 @@ public void testGetSuccessNoExistingTags() throws Exception { new TagAssociation().setTag(TagUrn.createFromString(TEST_TAG_2_URN)) ))); - final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); - proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); - proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal1.setAspectName(Constants.GLOBAL_TAGS_ASPECT_NAME); - proposal1.setAspect(GenericRecordUtils.serializeAspect(newTags)); - proposal1.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal1 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_1), + GLOBAL_TAGS_ASPECT_NAME, newTags); verifyIngestProposal(mockService, 1, proposal1); - final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); - proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); - proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal2.setAspectName(Constants.GLOBAL_TAGS_ASPECT_NAME); - proposal2.setAspect(GenericRecordUtils.serializeAspect(newTags)); - proposal2.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal2 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_2), + GLOBAL_TAGS_ASPECT_NAME, newTags); verifyIngestProposal(mockService, 1, proposal2); @@ -149,21 +141,13 @@ public void testGetSuccessExistingTags() throws Exception { new TagAssociation().setTag(TagUrn.createFromString(TEST_TAG_2_URN)) ))); - final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); - proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); - proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal1.setAspectName(Constants.GLOBAL_TAGS_ASPECT_NAME); - proposal1.setAspect(GenericRecordUtils.serializeAspect(newTags)); - proposal1.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal1 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_1), + GLOBAL_TAGS_ASPECT_NAME, newTags); verifyIngestProposal(mockService, 1, proposal1); - final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); - proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); - proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal2.setAspectName(Constants.GLOBAL_TAGS_ASPECT_NAME); - proposal2.setAspect(GenericRecordUtils.serializeAspect(newTags)); - proposal2.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal2 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_2), + GLOBAL_TAGS_ASPECT_NAME, newTags); verifyIngestProposal(mockService, 1, proposal2); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchRemoveTagsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchRemoveTagsResolverTest.java index 7d3d87636142a..44160cfbe1273 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchRemoveTagsResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchRemoveTagsResolverTest.java @@ -12,6 +12,7 @@ import com.linkedin.datahub.graphql.generated.BatchRemoveTagsInput; import com.linkedin.datahub.graphql.generated.ResourceRefInput; import com.linkedin.datahub.graphql.resolvers.mutate.BatchRemoveTagsResolver; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; @@ -24,6 +25,7 @@ import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.metadata.Constants.*; import static org.testng.Assert.*; @@ -72,19 +74,16 @@ public void testGetSuccessNoExistingTags() throws Exception { final GlobalTags emptyTags = new GlobalTags().setTags(new TagAssociationArray(Collections.emptyList())); - final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); - proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); - proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal1.setAspectName(Constants.GLOBAL_TAGS_ASPECT_NAME); - proposal1.setAspect(GenericRecordUtils.serializeAspect(emptyTags)); - proposal1.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal1 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_1), + GLOBAL_TAGS_ASPECT_NAME, emptyTags); Mockito.verify(mockService, Mockito.times(1)).ingestProposal( Mockito.eq(proposal1), Mockito.any(AuditStamp.class), Mockito.eq(false) ); - final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); + final MetadataChangeProposal proposal2 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_2), + GLOBAL_TAGS_ASPECT_NAME, emptyTags); proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); proposal2.setAspectName(Constants.GLOBAL_TAGS_ASPECT_NAME); @@ -142,24 +141,16 @@ public void testGetSuccessExistingTags() throws Exception { final GlobalTags emptyTags = new GlobalTags().setTags(new TagAssociationArray(Collections.emptyList())); - final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); - proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); - proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal1.setAspectName(Constants.GLOBAL_TAGS_ASPECT_NAME); - proposal1.setAspect(GenericRecordUtils.serializeAspect(emptyTags)); - proposal1.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal1 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_1), + GLOBAL_TAGS_ASPECT_NAME, emptyTags); Mockito.verify(mockService, Mockito.times(1)).ingestProposal( Mockito.eq(proposal1), Mockito.any(AuditStamp.class), Mockito.eq(false) ); - final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); - proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); - proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); - proposal2.setAspectName(Constants.GLOBAL_TAGS_ASPECT_NAME); - proposal2.setAspect(GenericRecordUtils.serializeAspect(emptyTags)); - proposal2.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal2 = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(TEST_ENTITY_URN_2), + GLOBAL_TAGS_ASPECT_NAME, emptyTags); verifyIngestProposal(mockService, 1, proposal2); } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolverTest.java index ebf992913af09..d294f806d1af7 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolverTest.java @@ -3,13 +3,11 @@ import com.datahub.authentication.Authentication; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.CreateTagInput; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.entity.EntityService; import com.linkedin.tag.TagProperties; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.key.TagKey; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetchingEnvironment; import java.util.concurrent.CompletionException; @@ -18,6 +16,7 @@ import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.metadata.Constants.*; import static org.testng.Assert.*; @@ -48,20 +47,17 @@ public void testGetSuccess() throws Exception { final TagKey key = new TagKey(); key.setName("test-id"); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); - proposal.setEntityType(Constants.TAG_ENTITY_NAME); TagProperties props = new TagProperties(); props.setDescription("test-description"); props.setName("test-name"); - proposal.setAspectName(Constants.TAG_PROPERTIES_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(props)); - proposal.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal = MutationUtils.buildMetadataChangeProposalWithKey(key, TAG_ENTITY_NAME, + TAG_PROPERTIES_ASPECT_NAME, props); // Not ideal to match against "any", but we don't know the auto-generated execution request id Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( Mockito.eq(proposal), - Mockito.any(Authentication.class) + Mockito.any(Authentication.class), + Mockito.eq(false) ); } @@ -91,7 +87,8 @@ public void testGetEntityClientException() throws Exception { EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.doThrow(RuntimeException.class).when(mockClient).ingestProposal( Mockito.any(), - Mockito.any(Authentication.class)); + Mockito.any(Authentication.class), + Mockito.eq(false)); CreateTagResolver resolver = new CreateTagResolver(mockClient, mockService); // Execute resolver diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/SetTagColorResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/SetTagColorResolverTest.java index d02e584b092bf..da474ca3e0e56 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/SetTagColorResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/SetTagColorResolverTest.java @@ -4,16 +4,16 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.entity.Aspect; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; import com.linkedin.tag.TagProperties; @@ -24,6 +24,7 @@ import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.metadata.Constants.*; import static org.testng.Assert.*; @@ -59,16 +60,13 @@ public void testGetSuccessExistingProperties() throws Exception { resolver.get(mockEnv).get(); final TagProperties newTagProperties = new TagProperties().setName("Test Tag").setColorHex(TEST_COLOR_HEX); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN)); - proposal.setEntityType(Constants.TAG_ENTITY_NAME); - proposal.setAspectName(Constants.TAG_PROPERTIES_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(newTagProperties)); - proposal.setChangeType(ChangeType.UPSERT); + final MetadataChangeProposal proposal = MutationUtils.buildMetadataChangeProposalWithUrn(UrnUtils.getUrn(TEST_ENTITY_URN), + TAG_PROPERTIES_ASPECT_NAME, newTagProperties); Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( Mockito.eq(proposal), - Mockito.any(Authentication.class) + Mockito.any(Authentication.class), + Mockito.eq(false) ); Mockito.verify(mockService, Mockito.times(1)).exists( diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/test/CreateTestResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/test/CreateTestResolverTest.java index d39c8b09743cc..911152d8c97c1 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/test/CreateTestResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/test/CreateTestResolverTest.java @@ -11,11 +11,11 @@ import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; -import com.linkedin.test.TestDefinition; import com.linkedin.test.TestDefinitionType; import com.linkedin.test.TestInfo; import graphql.schema.DataFetchingEnvironment; import java.util.concurrent.CompletionException; +import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.testng.annotations.Test; @@ -50,23 +50,21 @@ public void testGetSuccess() throws Exception { final TestKey key = new TestKey(); key.setId("test-id"); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); - proposal.setEntityType(Constants.TEST_ENTITY_NAME); - TestInfo info = new TestInfo(); - info.setCategory("test-category"); - info.setDescription("test-description"); - info.setName("test-name"); - info.setDefinition(new TestDefinition().setJson("{}").setType(TestDefinitionType.JSON)); - proposal.setAspectName(Constants.TEST_INFO_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(info)); - proposal.setChangeType(ChangeType.UPSERT); - - // Not ideal to match against "any", but we don't know the auto-generated execution request id - Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( - Mockito.eq(proposal), - Mockito.any(Authentication.class) - ); + ArgumentCaptor proposalCaptor = ArgumentCaptor.forClass(MetadataChangeProposal.class); + Mockito.verify(mockClient, Mockito.times(1)) + .ingestProposal(proposalCaptor.capture(), Mockito.any(Authentication.class), Mockito.eq(false)); + MetadataChangeProposal resultProposal = proposalCaptor.getValue(); + assertEquals(resultProposal.getEntityType(), Constants.TEST_ENTITY_NAME); + assertEquals(resultProposal.getAspectName(), Constants.TEST_INFO_ASPECT_NAME); + assertEquals(resultProposal.getChangeType(), ChangeType.UPSERT); + assertEquals(resultProposal.getEntityKeyAspect(), GenericRecordUtils.serializeAspect(key)); + TestInfo resultInfo = GenericRecordUtils.deserializeAspect(resultProposal.getAspect().getValue(), + resultProposal.getAspect().getContentType(), TestInfo.class); + assertEquals(resultInfo.getName(), "test-name"); + assertEquals(resultInfo.getCategory(), "test-category"); + assertEquals(resultInfo.getDescription(), "test-description"); + assertEquals(resultInfo.getDefinition().getType(), TestDefinitionType.JSON); + assertEquals(resultInfo.getDefinition().getJson(), "{}"); } @Test @@ -93,7 +91,8 @@ public void testGetEntityClientException() throws Exception { EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.doThrow(RemoteInvocationException.class).when(mockClient).ingestProposal( Mockito.any(), - Mockito.any(Authentication.class)); + Mockito.any(Authentication.class), + Mockito.eq(false)); CreateTestResolver resolver = new CreateTestResolver(mockClient); // Execute resolver @@ -104,4 +103,4 @@ public void testGetEntityClientException() throws Exception { assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); } -} \ No newline at end of file +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/test/UpdateTestResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/test/UpdateTestResolverTest.java index ea63c14760829..ae24232bce17c 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/test/UpdateTestResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/test/UpdateTestResolverTest.java @@ -11,11 +11,11 @@ import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; -import com.linkedin.test.TestDefinition; import com.linkedin.test.TestDefinitionType; import com.linkedin.test.TestInfo; import graphql.schema.DataFetchingEnvironment; import java.util.concurrent.CompletionException; +import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.testng.annotations.Test; @@ -48,23 +48,21 @@ public void testGetSuccess() throws Exception { resolver.get(mockEnv).get(); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(UrnUtils.getUrn(TEST_URN)); - proposal.setEntityType(Constants.TEST_ENTITY_NAME); - TestInfo info = new TestInfo(); - info.setCategory("test-category"); - info.setDescription("test-description"); - info.setName("test-name"); - info.setDefinition(new TestDefinition().setJson("{}").setType(TestDefinitionType.JSON)); - proposal.setAspectName(Constants.TEST_INFO_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(info)); - proposal.setChangeType(ChangeType.UPSERT); - - // Not ideal to match against "any", but we don't know the auto-generated execution request id - Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( - Mockito.eq(proposal), - Mockito.any(Authentication.class) - ); + ArgumentCaptor proposalCaptor = ArgumentCaptor.forClass(MetadataChangeProposal.class); + Mockito.verify(mockClient, Mockito.times(1)) + .ingestProposal(proposalCaptor.capture(), Mockito.any(Authentication.class), Mockito.eq(false)); + MetadataChangeProposal resultProposal = proposalCaptor.getValue(); + assertEquals(resultProposal.getEntityType(), Constants.TEST_ENTITY_NAME); + assertEquals(resultProposal.getAspectName(), Constants.TEST_INFO_ASPECT_NAME); + assertEquals(resultProposal.getChangeType(), ChangeType.UPSERT); + assertEquals(resultProposal.getEntityUrn(), UrnUtils.getUrn(TEST_URN)); + TestInfo resultInfo = GenericRecordUtils.deserializeAspect(resultProposal.getAspect().getValue(), + resultProposal.getAspect().getContentType(), TestInfo.class); + assertEquals(resultInfo.getName(), "test-name"); + assertEquals(resultInfo.getCategory(), "test-category"); + assertEquals(resultInfo.getDescription(), "test-description"); + assertEquals(resultInfo.getDefinition().getType(), TestDefinitionType.JSON); + assertEquals(resultInfo.getDefinition().getJson(), "{}"); } @Test @@ -103,4 +101,4 @@ public void testGetEntityClientException() throws Exception { assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); } -} \ No newline at end of file +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/MutationsUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/MutationsUtilsTest.java new file mode 100644 index 0000000000000..48ce2ddb6dde4 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/MutationsUtilsTest.java @@ -0,0 +1,25 @@ +package com.linkedin.datahub.graphql.utils; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; +import com.linkedin.identity.CorpUserInfo; +import com.linkedin.metadata.key.CorpUserKey; +import com.linkedin.mxe.MetadataChangeProposal; +import org.testng.annotations.Test; + +import static com.linkedin.metadata.Constants.*; +import static org.testng.AssertJUnit.*; + + +public class MutationsUtilsTest { + + @Test + public void testBuildMetadataChangeProposal() { + MetadataChangeProposal metadataChangeProposal = MutationUtils.buildMetadataChangeProposalWithUrn( + UrnUtils.getUrn("urn:li:corpuser:datahub"), CORP_USER_INFO_ASPECT_NAME, new CorpUserInfo().setActive(true)); + assertEquals(UI_SOURCE, metadataChangeProposal.getSystemMetadata().getProperties().get(APP_SOURCE)); + metadataChangeProposal = MutationUtils.buildMetadataChangeProposalWithKey(new CorpUserKey().setUsername("datahub"), + CORP_USER_ENTITY_NAME, CORP_USER_INFO_ASPECT_NAME, new CorpUserInfo().setActive(true)); + assertEquals(UI_SOURCE, metadataChangeProposal.getSystemMetadata().getProperties().get(APP_SOURCE)); + } +} diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java index d0959c3b5a59e..781582ecb2482 100644 --- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -28,6 +28,11 @@ public class Constants { public static final String DEFAULT_RUN_ID = "no-run-id-provided"; // Forces indexing for no-ops, enabled for restore indices calls. Only considered in the no-op case public static final String FORCE_INDEXING_KEY = "forceIndexing"; + // Indicates an event source from an application with hooks that have already been processed and should not be reprocessed + public static final String APP_SOURCE = "appSource"; + + // App sources + public static final String UI_SOURCE = "ui"; /** * Entities diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/PreProcessHooks.java b/metadata-io/src/main/java/com/linkedin/metadata/config/PreProcessHooks.java new file mode 100644 index 0000000000000..912abf525631b --- /dev/null +++ b/metadata-io/src/main/java/com/linkedin/metadata/config/PreProcessHooks.java @@ -0,0 +1,9 @@ +package com.linkedin.metadata.config; + +import lombok.Data; + + +@Data +public class PreProcessHooks { + private boolean uiEnabled; +} diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java index c09acc0b39cbf..fca9960d94994 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java @@ -22,7 +22,6 @@ import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.common.urn.VersionedUrnUtils; -import com.linkedin.data.DataMap; import com.linkedin.data.schema.RecordDataSchema; import com.linkedin.data.schema.TyperefDataSchema; import com.linkedin.data.schema.validator.Validator; @@ -41,6 +40,7 @@ import com.linkedin.metadata.Constants; import com.linkedin.metadata.aspect.Aspect; import com.linkedin.metadata.aspect.VersionedAspect; +import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.entity.ebean.EbeanAspectV2; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesResult; @@ -57,6 +57,7 @@ import com.linkedin.metadata.models.registry.template.AspectTemplateEngine; import com.linkedin.metadata.query.ListUrnsResult; import com.linkedin.metadata.run.AspectRowSummary; +import com.linkedin.metadata.service.UpdateIndicesService; import com.linkedin.metadata.snapshot.Snapshot; import com.linkedin.metadata.utils.DataPlatformInstanceUtils; import com.linkedin.metadata.utils.EntityKeyUtils; @@ -169,6 +170,8 @@ public static class IngestProposalResult { private final Map> _entityToValidAspects; private RetentionService _retentionService; private final Boolean _alwaysEmitChangeLog; + private final UpdateIndicesService _updateIndicesService; + private final PreProcessHooks _preProcessHooks; public static final String DEFAULT_RUN_ID = "no-run-id-provided"; public static final String BROWSE_PATHS = "browsePaths"; public static final String DATA_PLATFORM_INSTANCE = "dataPlatformInstance"; @@ -183,13 +186,17 @@ public EntityService( @Nonnull final AspectDao aspectDao, @Nonnull final EventProducer producer, @Nonnull final EntityRegistry entityRegistry, - final boolean alwaysEmitChangeLog) { + final boolean alwaysEmitChangeLog, + final UpdateIndicesService updateIndicesService, + final PreProcessHooks preProcessHooks) { _aspectDao = aspectDao; _producer = producer; _entityRegistry = entityRegistry; _entityToValidAspects = buildEntityToValidAspects(entityRegistry); _alwaysEmitChangeLog = alwaysEmitChangeLog; + _updateIndicesService = updateIndicesService; + _preProcessHooks = preProcessHooks; } /** @@ -1053,28 +1060,14 @@ private boolean emitChangeLog(@Nullable RecordTemplate oldAspect, @Nullable Syst if (!isNoOp || _alwaysEmitChangeLog || shouldAspectEmitChangeLog(aspectSpec)) { log.debug("Producing MetadataChangeLog for ingested aspect {}, urn {}", mcp.getAspectName(), entityUrn); - // Uses new data map to prevent side effects on original - final MetadataChangeLog metadataChangeLog = new MetadataChangeLog(new DataMap(mcp.data())); - metadataChangeLog.setEntityUrn(entityUrn); - metadataChangeLog.setCreated(auditStamp); - metadataChangeLog.setChangeType(isNoOp ? ChangeType.RESTATE : ChangeType.UPSERT); - - if (oldAspect != null) { - metadataChangeLog.setPreviousAspectValue(GenericRecordUtils.serializeAspect(oldAspect)); - } - if (oldSystemMetadata != null) { - metadataChangeLog.setPreviousSystemMetadata(oldSystemMetadata); - } - if (newAspect != null) { - metadataChangeLog.setAspect(GenericRecordUtils.serializeAspect(newAspect)); - } - if (newSystemMetadata != null) { - metadataChangeLog.setSystemMetadata(newSystemMetadata); - } + final MetadataChangeLog metadataChangeLog = constructMCL(mcp, urnToEntityName(entityUrn), entityUrn, + isNoOp ? ChangeType.RESTATE : ChangeType.UPSERT, aspectSpec.getName(), auditStamp, newAspect, newSystemMetadata, + oldAspect, oldSystemMetadata); log.debug("Serialized MCL event: {}", metadataChangeLog); produceMetadataChangeLog(entityUrn, aspectSpec, metadataChangeLog); + preprocessEvent(metadataChangeLog); return true; } else { @@ -1085,6 +1078,19 @@ private boolean emitChangeLog(@Nullable RecordTemplate oldAspect, @Nullable Syst } } + private void preprocessEvent(MetadataChangeLog metadataChangeLog) { + if (_preProcessHooks.isUiEnabled()) { + if (metadataChangeLog.getSystemMetadata() != null) { + if (metadataChangeLog.getSystemMetadata().getProperties() != null) { + if (UI_SOURCE.equals(metadataChangeLog.getSystemMetadata().getProperties().get(APP_SOURCE))) { + // Pre-process the update indices hook for UI updates to avoid perceived lag from Kafka + _updateIndicesService.handleChangeEvent(metadataChangeLog); + } + } + } + } + } + public Integer getCountAspect(@Nonnull String aspectName, @Nullable String urnLike) { return _aspectDao.countAspect(aspectName, urnLike); } @@ -1326,24 +1332,8 @@ public void produceMetadataChangeLog(@Nonnull final Urn urn, @Nonnull String ent @Nonnull final AspectSpec aspectSpec, @Nullable final RecordTemplate oldAspectValue, @Nullable final RecordTemplate newAspectValue, @Nullable final SystemMetadata oldSystemMetadata, @Nullable final SystemMetadata newSystemMetadata, @Nonnull AuditStamp auditStamp, @Nonnull final ChangeType changeType) { - final MetadataChangeLog metadataChangeLog = new MetadataChangeLog(); - metadataChangeLog.setEntityType(entityName); - metadataChangeLog.setEntityUrn(urn); - metadataChangeLog.setChangeType(changeType); - metadataChangeLog.setAspectName(aspectName); - metadataChangeLog.setCreated(auditStamp); - if (newAspectValue != null) { - metadataChangeLog.setAspect(GenericRecordUtils.serializeAspect(newAspectValue)); - } - if (newSystemMetadata != null) { - metadataChangeLog.setSystemMetadata(newSystemMetadata); - } - if (oldAspectValue != null) { - metadataChangeLog.setPreviousAspectValue(GenericRecordUtils.serializeAspect(oldAspectValue)); - } - if (oldSystemMetadata != null) { - metadataChangeLog.setPreviousSystemMetadata(oldSystemMetadata); - } + final MetadataChangeLog metadataChangeLog = constructMCL(null, entityName, urn, changeType, aspectName, auditStamp, + newAspectValue, newSystemMetadata, oldAspectValue, oldSystemMetadata); produceMetadataChangeLog(urn, aspectSpec, metadataChangeLog); } diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/GraphIndexUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/GraphIndexUtils.java similarity index 98% rename from metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/GraphIndexUtils.java rename to metadata-io/src/main/java/com/linkedin/metadata/graph/GraphIndexUtils.java index 6ab7a4b766531..2bbf2af1437d8 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/GraphIndexUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/GraphIndexUtils.java @@ -1,10 +1,9 @@ -package com.linkedin.metadata.kafka.hook; +package com.linkedin.metadata.graph; import com.datahub.util.RecordUtils; import com.linkedin.common.urn.Urn; import com.linkedin.data.schema.PathSpec; import com.linkedin.data.template.RecordTemplate; -import com.linkedin.metadata.graph.Edge; import com.linkedin.metadata.models.RelationshipFieldSpec; import com.linkedin.mxe.MetadataChangeLog; import com.linkedin.mxe.SystemMetadata; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java new file mode 100644 index 0000000000000..36b685f084d51 --- /dev/null +++ b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java @@ -0,0 +1,554 @@ +package com.linkedin.metadata.service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.InputField; +import com.linkedin.common.InputFields; +import com.linkedin.common.Status; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.dataset.FineGrainedLineage; +import com.linkedin.dataset.UpstreamLineage; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.graph.Edge; +import com.linkedin.metadata.graph.GraphIndexUtils; +import com.linkedin.metadata.graph.GraphService; +import com.linkedin.metadata.graph.dgraph.DgraphGraphService; +import com.linkedin.metadata.key.SchemaFieldKey; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.metadata.models.RelationshipFieldSpec; +import com.linkedin.metadata.models.extractor.FieldExtractor; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.query.filter.RelationshipDirection; +import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.search.transformer.SearchDocumentTransformer; +import com.linkedin.metadata.search.utils.SearchUtils; +import com.linkedin.metadata.systemmetadata.SystemMetadataService; +import com.linkedin.metadata.timeseries.TimeseriesAspectService; +import com.linkedin.metadata.timeseries.transformer.TimeseriesAspectTransformer; +import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.GenericAspect; +import com.linkedin.mxe.MetadataChangeLog; +import com.linkedin.mxe.SystemMetadata; +import com.linkedin.util.Pair; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; + +import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.search.utils.QueryUtils.*; + +@Slf4j +public class UpdateIndicesService { + private static final String DOWNSTREAM_OF = "DownstreamOf"; + + private final GraphService _graphService; + private final EntitySearchService _entitySearchService; + private final TimeseriesAspectService _timeseriesAspectService; + private final SystemMetadataService _systemMetadataService; + private final EntityRegistry _entityRegistry; + private final SearchDocumentTransformer _searchDocumentTransformer; + + @Value("${featureFlags.graphServiceDiffModeEnabled:true}") + private boolean _graphDiffMode; + @Value("${featureFlags.searchServiceDiffModeEnabled:true}") + private boolean _searchDiffMode; + + private static final Set UPDATE_CHANGE_TYPES = ImmutableSet.of( + ChangeType.UPSERT, + ChangeType.RESTATE, + ChangeType.PATCH); + + @VisibleForTesting + public void setGraphDiffMode(boolean graphDiffMode) { + _graphDiffMode = graphDiffMode; + } + + @VisibleForTesting + public void setSearchDiffMode(boolean searchDiffMode) { + _searchDiffMode = searchDiffMode; + } + + public UpdateIndicesService( + GraphService graphService, + EntitySearchService entitySearchService, + TimeseriesAspectService timeseriesAspectService, + SystemMetadataService systemMetadataService, + EntityRegistry entityRegistry, + SearchDocumentTransformer searchDocumentTransformer) { + _graphService = graphService; + _entitySearchService = entitySearchService; + _timeseriesAspectService = timeseriesAspectService; + _systemMetadataService = systemMetadataService; + _entityRegistry = entityRegistry; + _searchDocumentTransformer = searchDocumentTransformer; + } + + public void handleChangeEvent(@Nonnull final MetadataChangeLog event) { + if (UPDATE_CHANGE_TYPES.contains(event.getChangeType())) { + handleUpdateChangeEvent(event); + } else if (event.getChangeType() == ChangeType.DELETE) { + handleDeleteChangeEvent(event); + } + } + + + /** + * This very important method processes {@link MetadataChangeLog} events + * that represent changes to the Metadata Graph. + * + * In particular, it handles updating the Search, Graph, Timeseries, and + * System Metadata stores in response to a given change type to reflect + * the changes present in the new aspect. + * + * @param event the change event to be processed. + */ + public void handleUpdateChangeEvent(@Nonnull final MetadataChangeLog event) { + + final EntitySpec entitySpec = getEventEntitySpec(event); + final Urn urn = EntityKeyUtils.getUrnFromLog(event, entitySpec.getKeyAspectSpec()); + + if (!event.hasAspectName() || !event.hasAspect()) { + log.error("Aspect or aspect name is missing. Skipping aspect processing..."); + return; + } + + AspectSpec aspectSpec = entitySpec.getAspectSpec(event.getAspectName()); + if (aspectSpec == null) { + throw new RuntimeException( + String.format("Failed to retrieve Aspect Spec for entity with name %s, aspect with name %s. Cannot update indices for MCL.", + event.getEntityType(), + event.getAspectName())); + } + + RecordTemplate aspect = GenericRecordUtils.deserializeAspect( + event.getAspect().getValue(), + event.getAspect().getContentType(), + aspectSpec); + GenericAspect previousAspectValue = event.getPreviousAspectValue(); + RecordTemplate previousAspect = previousAspectValue != null + ? GenericRecordUtils.deserializeAspect(previousAspectValue.getValue(), previousAspectValue.getContentType(), aspectSpec) + : null; + + // Step 0. If the aspect is timeseries, add to its timeseries index. + if (aspectSpec.isTimeseries()) { + updateTimeseriesFields(event.getEntityType(), event.getAspectName(), urn, aspect, aspectSpec, + event.getSystemMetadata()); + } else { + // Inject into the System Metadata Index when an aspect is non-timeseries only. + // TODO: Verify whether timeseries aspects can be dropped into System Metadata as well + // without impacting rollbacks. + updateSystemMetadata(event.getSystemMetadata(), urn, aspectSpec, aspect); + } + + // Step 1. For all aspects, attempt to update Search + updateSearchService(entitySpec.getName(), urn, aspectSpec, aspect, event.getSystemMetadata(), previousAspect); + + // Step 2. For all aspects, attempt to update Graph + SystemMetadata systemMetadata = event.getSystemMetadata(); + if (_graphDiffMode && !(_graphService instanceof DgraphGraphService) + && (systemMetadata == null || systemMetadata.getProperties() == null + || !Boolean.parseBoolean(systemMetadata.getProperties().get(FORCE_INDEXING_KEY)))) { + updateGraphServiceDiff(urn, aspectSpec, previousAspect, aspect, event); + } else { + updateGraphService(urn, aspectSpec, aspect, event); + } + } + + /** + * This very important method processes {@link MetadataChangeLog} deletion events + * to cleanup the Metadata Graph when an aspect or entity is removed. + * + * In particular, it handles updating the Search, Graph, Timeseries, and + * System Metadata stores to reflect the deletion of a particular aspect. + * + * Note that if an entity's key aspect is deleted, the entire entity will be purged + * from search, graph, timeseries, etc. + * + * @param event the change event to be processed. + */ + public void handleDeleteChangeEvent(@Nonnull final MetadataChangeLog event) { + + final EntitySpec entitySpec = getEventEntitySpec(event); + final Urn urn = EntityKeyUtils.getUrnFromLog(event, entitySpec.getKeyAspectSpec()); + + if (!event.hasAspectName() || !event.hasPreviousAspectValue()) { + log.error("Previous aspect or aspect name is missing. Skipping aspect processing..."); + return; + } + + AspectSpec aspectSpec = entitySpec.getAspectSpec(event.getAspectName()); + if (aspectSpec == null) { + throw new RuntimeException( + String.format("Failed to retrieve Aspect Spec for entity with name %s, aspect with name %s. Cannot update indices for MCL.", + event.getEntityType(), + event.getAspectName())); + } + + RecordTemplate aspect = GenericRecordUtils.deserializeAspect(event.getPreviousAspectValue().getValue(), + event.getPreviousAspectValue().getContentType(), aspectSpec); + Boolean isDeletingKey = event.getAspectName().equals(entitySpec.getKeyAspectName()); + + if (!aspectSpec.isTimeseries()) { + deleteSystemMetadata(urn, aspectSpec, isDeletingKey); + deleteGraphData(urn, aspectSpec, aspect, isDeletingKey, event); + deleteSearchData(urn, entitySpec.getName(), aspectSpec, aspect, isDeletingKey); + } + } + + // TODO: remove this method once we implement sourceOverride when creating graph edges + private void updateFineGrainedEdgesAndRelationships( + RecordTemplate aspect, + List edgesToAdd, + HashMap> urnToRelationshipTypesBeingAdded + ) { + UpstreamLineage upstreamLineage = new UpstreamLineage(aspect.data()); + if (upstreamLineage.getFineGrainedLineages() != null) { + for (FineGrainedLineage fineGrainedLineage : upstreamLineage.getFineGrainedLineages()) { + if (!fineGrainedLineage.hasDownstreams() || !fineGrainedLineage.hasUpstreams()) { + break; + } + // for every downstream, create an edge with each of the upstreams + for (Urn downstream : fineGrainedLineage.getDownstreams()) { + for (Urn upstream : fineGrainedLineage.getUpstreams()) { + // TODO: add edges uniformly across aspects + edgesToAdd.add(new Edge(downstream, upstream, DOWNSTREAM_OF, null, null, null, null, null)); + Set relationshipTypes = urnToRelationshipTypesBeingAdded.getOrDefault(downstream, new HashSet<>()); + relationshipTypes.add(DOWNSTREAM_OF); + urnToRelationshipTypesBeingAdded.put(downstream, relationshipTypes); + } + } + } + } + } + + private Urn generateSchemaFieldUrn(@Nonnull final String resourceUrn, @Nonnull final String fieldPath) { + // we rely on schemaField fieldPaths to be encoded since we do that with fineGrainedLineage on the ingestion side + final String encodedFieldPath = fieldPath.replaceAll("\\(", "%28").replaceAll("\\)", "%29").replaceAll(",", "%2C"); + final SchemaFieldKey key = new SchemaFieldKey().setParent(UrnUtils.getUrn(resourceUrn)).setFieldPath(encodedFieldPath); + return EntityKeyUtils.convertEntityKeyToUrn(key, Constants.SCHEMA_FIELD_ENTITY_NAME); + } + + // TODO: remove this method once we implement sourceOverride and update inputFields aspect + private void updateInputFieldEdgesAndRelationships( + @Nonnull final Urn urn, + @Nonnull final InputFields inputFields, + @Nonnull final List edgesToAdd, + @Nonnull final HashMap> urnToRelationshipTypesBeingAdded + ) { + if (inputFields.hasFields()) { + for (final InputField field : inputFields.getFields()) { + if (field.hasSchemaFieldUrn() && field.hasSchemaField() && field.getSchemaField().hasFieldPath()) { + final Urn sourceFieldUrn = generateSchemaFieldUrn(urn.toString(), field.getSchemaField().getFieldPath()); + // TODO: add edges uniformly across aspects + edgesToAdd.add(new Edge(sourceFieldUrn, field.getSchemaFieldUrn(), DOWNSTREAM_OF, null, null, null, null, null)); + final Set relationshipTypes = urnToRelationshipTypesBeingAdded.getOrDefault(sourceFieldUrn, new HashSet<>()); + relationshipTypes.add(DOWNSTREAM_OF); + urnToRelationshipTypesBeingAdded.put(sourceFieldUrn, relationshipTypes); + } + } + } + } + + private Pair, HashMap>> getEdgesAndRelationshipTypesFromAspect( + @Nonnull final Urn urn, + @Nonnull final AspectSpec aspectSpec, + @Nonnull final RecordTemplate aspect, + @Nonnull final MetadataChangeLog event, + final boolean isNewAspectVersion + ) { + final List edgesToAdd = new ArrayList<>(); + final HashMap> urnToRelationshipTypesBeingAdded = new HashMap<>(); + + // we need to manually set schemaField <-> schemaField edges for fineGrainedLineage and inputFields + // since @Relationship only links between the parent entity urn and something else. + if (aspectSpec.getName().equals(Constants.UPSTREAM_LINEAGE_ASPECT_NAME)) { + updateFineGrainedEdgesAndRelationships(aspect, edgesToAdd, urnToRelationshipTypesBeingAdded); + } + if (aspectSpec.getName().equals(Constants.INPUT_FIELDS_ASPECT_NAME)) { + final InputFields inputFields = new InputFields(aspect.data()); + updateInputFieldEdgesAndRelationships(urn, inputFields, edgesToAdd, urnToRelationshipTypesBeingAdded); + } + + Map> extractedFields = + FieldExtractor.extractFields(aspect, aspectSpec.getRelationshipFieldSpecs()); + + for (Map.Entry> entry : extractedFields.entrySet()) { + Set relationshipTypes = urnToRelationshipTypesBeingAdded.getOrDefault(urn, new HashSet<>()); + relationshipTypes.add(entry.getKey().getRelationshipName()); + urnToRelationshipTypesBeingAdded.put(urn, relationshipTypes); + final List newEdges = GraphIndexUtils.extractGraphEdges(entry, aspect, urn, event, isNewAspectVersion); + edgesToAdd.addAll(newEdges); + } + return Pair.of(edgesToAdd, urnToRelationshipTypesBeingAdded); + } + + /** + * Process snapshot and update graph index + */ + private void updateGraphService( + @Nonnull final Urn urn, + @Nonnull final AspectSpec aspectSpec, + @Nonnull final RecordTemplate aspect, + @Nonnull final MetadataChangeLog event + ) { + Pair, HashMap>> edgeAndRelationTypes = + getEdgesAndRelationshipTypesFromAspect(urn, aspectSpec, aspect, event, true); + + final List edgesToAdd = edgeAndRelationTypes.getFirst(); + final HashMap> urnToRelationshipTypesBeingAdded = edgeAndRelationTypes.getSecond(); + + log.debug("Here's the relationship types found {}", urnToRelationshipTypesBeingAdded); + if (urnToRelationshipTypesBeingAdded.size() > 0) { + for (Map.Entry> entry : urnToRelationshipTypesBeingAdded.entrySet()) { + _graphService.removeEdgesFromNode(entry.getKey(), new ArrayList<>(entry.getValue()), + newRelationshipFilter(new Filter().setOr(new ConjunctiveCriterionArray()), RelationshipDirection.OUTGOING)); + } + edgesToAdd.forEach(_graphService::addEdge); + } + } + + private void updateGraphServiceDiff( + @Nonnull final Urn urn, + @Nonnull final AspectSpec aspectSpec, + @Nullable final RecordTemplate oldAspect, + @Nonnull final RecordTemplate newAspect, + @Nonnull final MetadataChangeLog event + ) { + Pair, HashMap>> oldEdgeAndRelationTypes = null; + if (oldAspect != null) { + oldEdgeAndRelationTypes = getEdgesAndRelationshipTypesFromAspect(urn, aspectSpec, oldAspect, event, false); + } + + final List oldEdges = + oldEdgeAndRelationTypes != null ? oldEdgeAndRelationTypes.getFirst() : Collections.emptyList(); + final Set oldEdgeSet = new HashSet<>(oldEdges); + + Pair, HashMap>> newEdgeAndRelationTypes = + getEdgesAndRelationshipTypesFromAspect(urn, aspectSpec, newAspect, event, true); + + final List newEdges = newEdgeAndRelationTypes.getFirst(); + final Set newEdgeSet = new HashSet<>(newEdges); + + // Edges to add + final List additiveDifference = newEdgeSet.stream() + .filter(edge -> !oldEdgeSet.contains(edge)) + .collect(Collectors.toList()); + + // Edges to remove + final List subtractiveDifference = oldEdgeSet.stream() + .filter(edge -> !newEdgeSet.contains(edge)) + .collect(Collectors.toList()); + + // Edges to update + final List mergedEdges = getMergedEdges(oldEdgeSet, newEdgeSet); + + // Remove any old edges that no longer exist first + if (subtractiveDifference.size() > 0) { + log.debug("Removing edges: {}", subtractiveDifference); + subtractiveDifference.forEach(_graphService::removeEdge); + } + + // Then add new edges + if (additiveDifference.size() > 0) { + log.debug("Adding edges: {}", additiveDifference); + additiveDifference.forEach(_graphService::addEdge); + } + + // Then update existing edges + if (mergedEdges.size() > 0) { + log.debug("Updating edges: {}", mergedEdges); + mergedEdges.forEach(_graphService::upsertEdge); + } + } + + private static List getMergedEdges(final Set oldEdgeSet, final Set newEdgeSet) { + final Map oldEdgesMap = oldEdgeSet + .stream() + .map(edge -> Pair.of(edge.hashCode(), edge)) + .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond)); + + final List mergedEdges = new ArrayList<>(); + if (!oldEdgesMap.isEmpty()) { + for (com.linkedin.metadata.graph.Edge newEdge : newEdgeSet) { + if (oldEdgesMap.containsKey(newEdge.hashCode())) { + final com.linkedin.metadata.graph.Edge oldEdge = oldEdgesMap.get(newEdge.hashCode()); + final com.linkedin.metadata.graph.Edge mergedEdge = GraphIndexUtils.mergeEdges(oldEdge, newEdge); + mergedEdges.add(mergedEdge); + } + } + } + + return mergedEdges; + } + + /** + * Process snapshot and update search index + */ + private void updateSearchService(String entityName, Urn urn, AspectSpec aspectSpec, RecordTemplate aspect, + @Nullable SystemMetadata systemMetadata, @Nullable RecordTemplate previousAspect) { + Optional searchDocument; + Optional previousSearchDocument = Optional.empty(); + try { + searchDocument = _searchDocumentTransformer.transformAspect(urn, aspect, aspectSpec, false); + } catch (Exception e) { + log.error("Error in getting documents from aspect: {} for aspect {}", e, aspectSpec.getName()); + return; + } + + if (!searchDocument.isPresent()) { + return; + } + + Optional docId = SearchUtils.getDocId(urn); + + if (!docId.isPresent()) { + return; + } + + String searchDocumentValue = searchDocument.get(); + if (_searchDiffMode && (systemMetadata == null || systemMetadata.getProperties() == null + || !Boolean.parseBoolean(systemMetadata.getProperties().get(FORCE_INDEXING_KEY)))) { + if (previousAspect != null) { + try { + previousSearchDocument = _searchDocumentTransformer.transformAspect(urn, previousAspect, aspectSpec, false); + } catch (Exception e) { + log.error( + "Error in getting documents from previous aspect state: {} for aspect {}, continuing without diffing.", e, + aspectSpec.getName()); + } + } + + if (previousSearchDocument.isPresent()) { + String previousSearchDocumentValue = previousSearchDocument.get(); + if (searchDocumentValue.equals(previousSearchDocumentValue)) { + // No changes to search document, skip writing no-op update + return; + } + } + } + + _entitySearchService.upsertDocument(entityName, searchDocument.get(), docId.get()); + } + + /** + * Process snapshot and update time-series index + */ + private void updateTimeseriesFields(String entityType, String aspectName, Urn urn, RecordTemplate aspect, + AspectSpec aspectSpec, SystemMetadata systemMetadata) { + Map documents; + try { + documents = TimeseriesAspectTransformer.transform(urn, aspect, aspectSpec, systemMetadata); + } catch (JsonProcessingException e) { + log.error("Failed to generate timeseries document from aspect: {}", e.toString()); + return; + } + documents.entrySet().forEach(document -> { + _timeseriesAspectService.upsertDocument(entityType, aspectName, document.getKey(), document.getValue()); + }); + } + + private void updateSystemMetadata(SystemMetadata systemMetadata, Urn urn, AspectSpec aspectSpec, RecordTemplate aspect) { + _systemMetadataService.insert(systemMetadata, urn.toString(), aspectSpec.getName()); + + // If processing status aspect update all aspects for this urn to removed + if (aspectSpec.getName().equals(Constants.STATUS_ASPECT_NAME)) { + _systemMetadataService.setDocStatus(urn.toString(), ((Status) aspect).isRemoved()); + } + } + + private void deleteSystemMetadata(Urn urn, AspectSpec aspectSpec, Boolean isKeyAspect) { + if (isKeyAspect) { + // Delete all aspects + log.debug(String.format("Deleting all system metadata for urn: %s", urn)); + _systemMetadataService.deleteUrn(urn.toString()); + } else { + // Delete all aspects from system metadata service + log.debug(String.format("Deleting system metadata for urn: %s, aspect: %s", urn, aspectSpec.getName())); + _systemMetadataService.deleteAspect(urn.toString(), aspectSpec.getName()); + } + } + + private void deleteGraphData( + @Nonnull final Urn urn, + @Nonnull final AspectSpec aspectSpec, + @Nonnull final RecordTemplate aspect, + @Nonnull final Boolean isKeyAspect, + @Nonnull final MetadataChangeLog event + ) { + if (isKeyAspect) { + _graphService.removeNode(urn); + return; + } + + Pair, HashMap>> edgeAndRelationTypes = + getEdgesAndRelationshipTypesFromAspect(urn, aspectSpec, aspect, event, true); + + final HashMap> urnToRelationshipTypesBeingAdded = edgeAndRelationTypes.getSecond(); + if (urnToRelationshipTypesBeingAdded.size() > 0) { + for (Map.Entry> entry : urnToRelationshipTypesBeingAdded.entrySet()) { + _graphService.removeEdgesFromNode(entry.getKey(), new ArrayList<>(entry.getValue()), + createRelationshipFilter(new Filter().setOr(new ConjunctiveCriterionArray()), RelationshipDirection.OUTGOING)); + } + } + } + + private void deleteSearchData(Urn urn, String entityName, AspectSpec aspectSpec, RecordTemplate aspect, Boolean isKeyAspect) { + String docId; + try { + docId = URLEncoder.encode(urn.toString(), "UTF-8"); + } catch (UnsupportedEncodingException e) { + log.error("Failed to encode the urn with error: {}", e.toString()); + return; + } + + if (isKeyAspect) { + _entitySearchService.deleteDocument(entityName, docId); + return; + } + + Optional searchDocument; + try { + searchDocument = _searchDocumentTransformer.transformAspect(urn, aspect, aspectSpec, true); // TODO + } catch (Exception e) { + log.error("Error in getting documents from aspect: {} for aspect {}", e, aspectSpec.getName()); + return; + } + + if (!searchDocument.isPresent()) { + return; + } + + _entitySearchService.upsertDocument(entityName, searchDocument.get(), docId); + } + + private EntitySpec getEventEntitySpec(@Nonnull final MetadataChangeLog event) { + try { + return _entityRegistry.getEntitySpec(event.getEntityType()); + } catch (IllegalArgumentException e) { + throw new RuntimeException( + String.format("Failed to retrieve Entity Spec for entity with name %s. Cannot update indices for MCL.", + event.getEntityType())); + } + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/AspectUtilsTest.java b/metadata-io/src/test/java/com/linkedin/metadata/AspectUtilsTest.java index 89d33992d80dc..80ab6e985fc82 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/AspectUtilsTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/AspectUtilsTest.java @@ -5,6 +5,7 @@ import com.linkedin.common.urn.DatasetUrn; import com.linkedin.dataset.DatasetProperties; import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.entity.AspectUtils; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.TestEntityRegistry; @@ -42,7 +43,10 @@ public void testAdditionalChanges() { EbeanAspectDao aspectDao = new EbeanAspectDao(server); aspectDao.setConnectionValidated(true); EventProducer mockProducer = mock(EventProducer.class); - EntityService entityService = new EntityService(aspectDao, mockProducer, _testEntityRegistry, true); + PreProcessHooks preProcessHooks = new PreProcessHooks(); + preProcessHooks.setUiEnabled(true); + EntityService entityService = new EntityService(aspectDao, mockProducer, _testEntityRegistry, true, + null, preProcessHooks); MetadataChangeProposal proposal1 = new MetadataChangeProposal(); proposal1.setEntityUrn(new DatasetUrn(new DataPlatformUrn("platform"), "name", FabricType.PROD)); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/ESSampleDataFixture.java b/metadata-io/src/test/java/com/linkedin/metadata/ESSampleDataFixture.java index efa1eb92c7a9d..e0cb1983bbde3 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/ESSampleDataFixture.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/ESSampleDataFixture.java @@ -4,6 +4,7 @@ import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.client.JavaEntityClient; import com.linkedin.metadata.config.search.CustomConfiguration; +import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.config.search.ElasticSearchConfiguration; import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; @@ -184,8 +185,11 @@ protected EntityClient entityClient( AspectDao mockAspectDao = mock(AspectDao.class); when(mockAspectDao.batchGet(anySet())).thenReturn(Map.of(mock(EntityAspectIdentifier.class), mock(EntityAspect.class))); + PreProcessHooks preProcessHooks = new PreProcessHooks(); + preProcessHooks.setUiEnabled(true); return new JavaEntityClient( - new EntityService(mockAspectDao, null, entityRegistry, true), + new EntityService(mockAspectDao, null, entityRegistry, true, null, + preProcessHooks), null, entitySearchService, cachingEntitySearchService, diff --git a/metadata-io/src/test/java/com/linkedin/metadata/ESSearchLineageFixture.java b/metadata-io/src/test/java/com/linkedin/metadata/ESSearchLineageFixture.java index 09ab67fb6f521..419cdbfc4866a 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/ESSearchLineageFixture.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/ESSearchLineageFixture.java @@ -2,6 +2,7 @@ import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.client.JavaEntityClient; +import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.config.cache.SearchLineageCacheConfiguration; import com.linkedin.metadata.config.search.ElasticSearchConfiguration; import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; @@ -218,8 +219,11 @@ protected EntityClient entityClient( 1, false); + PreProcessHooks preProcessHooks = new PreProcessHooks(); + preProcessHooks.setUiEnabled(true); return new JavaEntityClient( - new EntityService(null, null, entityRegistry, true), + new EntityService(null, null, entityRegistry, true, null, + preProcessHooks), null, entitySearchService, cachingEntitySearchService, diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/AspectMigrationsDaoTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/AspectMigrationsDaoTest.java index e780e1375a445..9304080460fce 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/AspectMigrationsDaoTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/AspectMigrationsDaoTest.java @@ -9,6 +9,7 @@ import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.models.registry.EntityRegistryException; import com.linkedin.metadata.models.registry.MergedEntityRegistry; +import com.linkedin.metadata.service.UpdateIndicesService; import com.linkedin.metadata.snapshot.Snapshot; import java.util.ArrayList; import java.util.List; @@ -32,6 +33,7 @@ abstract public class AspectMigrationsDaoTest { protected EntityService _entityService; protected RetentionService _retentionService; + protected UpdateIndicesService _mockUpdateIndicesService; protected AspectMigrationsDaoTest() throws EntityRegistryException { _snapshotEntityRegistry = new TestEntityRegistry(); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraAspectMigrationsDaoTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraAspectMigrationsDaoTest.java index b1408b3048d86..7e80a67e304c3 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraAspectMigrationsDaoTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraAspectMigrationsDaoTest.java @@ -2,10 +2,12 @@ import com.datastax.oss.driver.api.core.CqlSession; import com.linkedin.metadata.CassandraTestUtils; +import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.entity.cassandra.CassandraAspectDao; import com.linkedin.metadata.entity.cassandra.CassandraRetentionService; import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.models.registry.EntityRegistryException; +import com.linkedin.metadata.service.UpdateIndicesService; import org.testcontainers.containers.CassandraContainer; import org.testng.Assert; import org.testng.annotations.AfterClass; @@ -44,7 +46,11 @@ private void configureComponents() { CassandraAspectDao dao = new CassandraAspectDao(session); dao.setConnectionValidated(true); _mockProducer = mock(EventProducer.class); - _entityService = new EntityService(dao, _mockProducer, _testEntityRegistry, true); + _mockUpdateIndicesService = mock(UpdateIndicesService.class); + PreProcessHooks preProcessHooks = new PreProcessHooks(); + preProcessHooks.setUiEnabled(true); + _entityService = new EntityService(dao, _mockProducer, _testEntityRegistry, true, _mockUpdateIndicesService, + preProcessHooks); _retentionService = new CassandraRetentionService(_entityService, session, 1000); _entityService.setRetentionService(_retentionService); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraEntityServiceTest.java index a9732cef3be71..6e062fd14fc21 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraEntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraEntityServiceTest.java @@ -7,6 +7,7 @@ import com.linkedin.metadata.AspectGenerationUtils; import com.linkedin.metadata.AspectIngestionUtils; import com.linkedin.metadata.CassandraTestUtils; +import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.entity.cassandra.CassandraAspectDao; import com.linkedin.metadata.entity.cassandra.CassandraRetentionService; import com.linkedin.metadata.event.EventProducer; @@ -14,6 +15,7 @@ import com.linkedin.metadata.models.registry.EntityRegistryException; import com.linkedin.metadata.query.ExtraInfo; import com.linkedin.metadata.query.ListUrnsResult; +import com.linkedin.metadata.service.UpdateIndicesService; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -64,7 +66,11 @@ private void configureComponents() { _aspectDao = new CassandraAspectDao(session); _aspectDao.setConnectionValidated(true); _mockProducer = mock(EventProducer.class); - _entityService = new EntityService(_aspectDao, _mockProducer, _testEntityRegistry, true); + _mockUpdateIndicesService = mock(UpdateIndicesService.class); + PreProcessHooks preProcessHooks = new PreProcessHooks(); + preProcessHooks.setUiEnabled(true); + _entityService = new EntityService(_aspectDao, _mockProducer, _testEntityRegistry, true, + _mockUpdateIndicesService, preProcessHooks); _retentionService = new CassandraRetentionService(_entityService, session, 1000); _entityService.setRetentionService(_retentionService); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java index 31317943d22e5..759680abe7b4d 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java @@ -9,6 +9,7 @@ import com.linkedin.entity.EntityResponse; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; +import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.entity.ebean.EbeanAspectDao; import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.graph.GraphService; @@ -18,6 +19,7 @@ import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.query.filter.RelationshipDirection; import com.linkedin.metadata.run.DeleteReferencesResponse; +import com.linkedin.metadata.service.UpdateIndicesService; import com.linkedin.metadata.snapshot.Snapshot; import com.linkedin.metadata.utils.AuditStampUtils; import com.linkedin.metadata.utils.SystemMetadataUtils; @@ -39,6 +41,7 @@ public class DeleteEntityServiceTest { protected GraphService _graphService = Mockito.mock(GraphService.class); protected DeleteEntityService _deleteEntityService; + protected UpdateIndicesService _mockUpdateIndicesService; protected EntityRegistry _entityRegistry; @@ -46,7 +49,11 @@ public DeleteEntityServiceTest() { _entityRegistry = new ConfigEntityRegistry(Snapshot.class.getClassLoader() .getResourceAsStream("entity-registry.yml")); _aspectDao = mock(EbeanAspectDao.class); - _entityService = new EntityService(_aspectDao, mock(EventProducer.class), _entityRegistry, true); + _mockUpdateIndicesService = mock(UpdateIndicesService.class); + PreProcessHooks preProcessHooks = new PreProcessHooks(); + preProcessHooks.setUiEnabled(true); + _entityService = new EntityService(_aspectDao, mock(EventProducer.class), _entityRegistry, true, + _mockUpdateIndicesService, preProcessHooks); _deleteEntityService = new DeleteEntityService(_entityService, _graphService); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanAspectMigrationsDaoTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanAspectMigrationsDaoTest.java index 7c6d5230d2d4e..2aa66ae7d9d3b 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanAspectMigrationsDaoTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanAspectMigrationsDaoTest.java @@ -1,10 +1,12 @@ package com.linkedin.metadata.entity; import com.linkedin.metadata.EbeanTestUtils; +import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.entity.ebean.EbeanAspectDao; import com.linkedin.metadata.entity.ebean.EbeanRetentionService; import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.models.registry.EntityRegistryException; +import com.linkedin.metadata.service.UpdateIndicesService; import io.ebean.EbeanServer; import org.testng.Assert; import org.testng.annotations.BeforeMethod; @@ -24,7 +26,11 @@ public void setupTest() { _mockProducer = mock(EventProducer.class); EbeanAspectDao dao = new EbeanAspectDao(server); dao.setConnectionValidated(true); - _entityService = new EntityService(dao, _mockProducer, _testEntityRegistry, true); + _mockUpdateIndicesService = mock(UpdateIndicesService.class); + PreProcessHooks preProcessHooks = new PreProcessHooks(); + preProcessHooks.setUiEnabled(true); + _entityService = new EntityService(dao, _mockProducer, _testEntityRegistry, true, + _mockUpdateIndicesService, preProcessHooks); _retentionService = new EbeanRetentionService(_entityService, server, 1000); _entityService.setRetentionService(_retentionService); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java index 33c6748311436..3d5b8c6cfaa2b 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java @@ -7,12 +7,14 @@ import com.linkedin.identity.CorpUserInfo; import com.linkedin.metadata.AspectGenerationUtils; import com.linkedin.metadata.EbeanTestUtils; +import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.entity.ebean.EbeanAspectDao; import com.linkedin.metadata.entity.ebean.EbeanRetentionService; import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.key.CorpUserKey; import com.linkedin.metadata.models.registry.EntityRegistryException; import com.linkedin.metadata.query.ListUrnsResult; +import com.linkedin.metadata.service.UpdateIndicesService; import com.linkedin.metadata.utils.PegasusUtils; import com.linkedin.mxe.SystemMetadata; import io.ebean.EbeanServer; @@ -46,7 +48,11 @@ public void setupTest() { _mockProducer = mock(EventProducer.class); _aspectDao = new EbeanAspectDao(server); _aspectDao.setConnectionValidated(true); - _entityService = new EntityService(_aspectDao, _mockProducer, _testEntityRegistry, true); + _mockUpdateIndicesService = mock(UpdateIndicesService.class); + PreProcessHooks preProcessHooks = new PreProcessHooks(); + preProcessHooks.setUiEnabled(true); + _entityService = new EntityService(_aspectDao, _mockProducer, _testEntityRegistry, true, + _mockUpdateIndicesService, preProcessHooks); _retentionService = new EbeanRetentionService(_entityService, server, 1000); _entityService.setRetentionService(_retentionService); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java index de5e080f750be..117ed92e6f468 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java @@ -20,6 +20,7 @@ import com.linkedin.data.template.StringMap; import com.linkedin.dataset.DatasetProfile; import com.linkedin.dataset.DatasetProperties; +import com.linkedin.dataset.EditableDatasetProperties; import com.linkedin.dataset.UpstreamLineage; import com.linkedin.entity.Entity; import com.linkedin.entity.EntityResponse; @@ -40,6 +41,7 @@ import com.linkedin.metadata.models.registry.EntityRegistryException; import com.linkedin.metadata.models.registry.MergedEntityRegistry; import com.linkedin.metadata.run.AspectRowSummary; +import com.linkedin.metadata.service.UpdateIndicesService; import com.linkedin.metadata.snapshot.CorpUserSnapshot; import com.linkedin.metadata.snapshot.Snapshot; import com.linkedin.metadata.utils.GenericRecordUtils; @@ -96,6 +98,7 @@ abstract public class EntityServiceTest captor = ArgumentCaptor.forClass(MetadataChangeLog.class); + verify(_mockProducer, times(1)).produceMetadataChangeLog(Mockito.eq(entityUrn), + Mockito.any(), captor.capture()); + assertEquals(UI_SOURCE, captor.getValue().getSystemMetadata().getProperties().get(APP_SOURCE)); + } + @Nonnull protected com.linkedin.entity.Entity createCorpUserEntity(Urn entityUrn, String email) throws Exception { CorpuserUrn corpuserUrn = CorpuserUrn.createFromUrn(entityUrn); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/timeline/CassandraTimelineServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/timeline/CassandraTimelineServiceTest.java index 00555d74e53fd..fd8cdec73a191 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/timeline/CassandraTimelineServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/timeline/CassandraTimelineServiceTest.java @@ -2,6 +2,7 @@ import com.datastax.oss.driver.api.core.CqlSession; import com.linkedin.metadata.CassandraTestUtils; +import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.cassandra.CassandraAspectDao; import com.linkedin.metadata.event.EventProducer; @@ -51,7 +52,10 @@ private void configureComponents() { _aspectDao.setConnectionValidated(true); _entityTimelineService = new TimelineServiceImpl(_aspectDao, _testEntityRegistry); _mockProducer = mock(EventProducer.class); - _entityService = new EntityService(_aspectDao, _mockProducer, _testEntityRegistry, true); + PreProcessHooks preProcessHooks = new PreProcessHooks(); + preProcessHooks.setUiEnabled(true); + _entityService = new EntityService(_aspectDao, _mockProducer, _testEntityRegistry, true, + _mockUpdateIndicesService, preProcessHooks); } /** diff --git a/metadata-io/src/test/java/com/linkedin/metadata/timeline/EbeanTimelineServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/timeline/EbeanTimelineServiceTest.java index 9650d156afeb8..0ea2f6bb124d9 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/timeline/EbeanTimelineServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/timeline/EbeanTimelineServiceTest.java @@ -1,6 +1,7 @@ package com.linkedin.metadata.timeline; import com.linkedin.metadata.EbeanTestUtils; +import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.ebean.EbeanAspectDao; import com.linkedin.metadata.event.EventProducer; @@ -31,7 +32,10 @@ public void setupTest() { _aspectDao.setConnectionValidated(true); _entityTimelineService = new TimelineServiceImpl(_aspectDao, _testEntityRegistry); _mockProducer = mock(EventProducer.class); - _entityService = new EntityService(_aspectDao, _mockProducer, _testEntityRegistry, true); + PreProcessHooks preProcessHooks = new PreProcessHooks(); + preProcessHooks.setUiEnabled(true); + _entityService = new EntityService(_aspectDao, _mockProducer, _testEntityRegistry, true, + _mockUpdateIndicesService, preProcessHooks); } /** diff --git a/metadata-io/src/test/java/com/linkedin/metadata/timeline/TimelineServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/timeline/TimelineServiceTest.java index 4bfeefb87a424..abd611c5a9336 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/timeline/TimelineServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/timeline/TimelineServiceTest.java @@ -14,6 +14,7 @@ import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.models.registry.EntityRegistryException; import com.linkedin.metadata.models.registry.MergedEntityRegistry; +import com.linkedin.metadata.service.UpdateIndicesService; import com.linkedin.metadata.snapshot.Snapshot; import com.linkedin.metadata.timeline.data.ChangeCategory; import com.linkedin.metadata.timeline.data.ChangeTransaction; @@ -35,6 +36,8 @@ import java.util.Map; import java.util.Set; +import static org.mockito.Mockito.*; + /** * A class to test {@link TimelineServiceImpl} @@ -60,6 +63,7 @@ abstract public class TimelineServiceTest { protected TimelineServiceImpl _entityTimelineService; protected EntityService _entityService; protected EventProducer _mockProducer; + protected UpdateIndicesService _mockUpdateIndicesService = mock(UpdateIndicesService.class); protected TimelineServiceTest() throws EntityRegistryException { } diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java index 4155048c9c797..0e47202a9d237 100644 --- a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java +++ b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java @@ -21,10 +21,11 @@ "com.linkedin.metadata.boot.kafka", "com.linkedin.metadata.kafka", "com.linkedin.metadata.dao.producer", - "com.linkedin.gms.factory.config" + "com.linkedin.gms.factory.config", + "com.linkedin.gms.factory.entity.update.indices" }, excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, classes = ScheduledAnalyticsFactory.class), -@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, classes = SiblingGraphServiceFactory.class)} + @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, classes = SiblingGraphServiceFactory.class)} ) public class MaeConsumerApplication { public static void main(String[] args) { diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHook.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHook.java index 6899a69b4260a..fad7a34074964 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHook.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHook.java @@ -1,70 +1,21 @@ package com.linkedin.metadata.kafka.hook; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableSet; -import com.linkedin.common.InputField; -import com.linkedin.common.InputFields; -import com.linkedin.common.Status; -import com.linkedin.common.urn.Urn; -import com.linkedin.common.urn.UrnUtils; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.dataset.FineGrainedLineage; -import com.linkedin.dataset.UpstreamLineage; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.gms.factory.common.GraphServiceFactory; import com.linkedin.gms.factory.common.SystemMetadataServiceFactory; import com.linkedin.gms.factory.entityregistry.EntityRegistryFactory; import com.linkedin.gms.factory.search.EntitySearchServiceFactory; import com.linkedin.gms.factory.search.SearchDocumentTransformerFactory; import com.linkedin.gms.factory.timeseries.TimeseriesAspectServiceFactory; -import com.linkedin.metadata.Constants; -import com.linkedin.metadata.graph.Edge; -import com.linkedin.metadata.graph.GraphService; -import com.linkedin.metadata.graph.dgraph.DgraphGraphService; -import com.linkedin.metadata.key.SchemaFieldKey; -import com.linkedin.metadata.models.AspectSpec; -import com.linkedin.metadata.models.EntitySpec; -import com.linkedin.metadata.models.RelationshipFieldSpec; -import com.linkedin.metadata.models.extractor.FieldExtractor; -import com.linkedin.metadata.models.registry.EntityRegistry; -import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; -import com.linkedin.metadata.query.filter.Filter; -import com.linkedin.metadata.query.filter.RelationshipDirection; -import com.linkedin.metadata.search.EntitySearchService; -import com.linkedin.metadata.search.transformer.SearchDocumentTransformer; -import com.linkedin.metadata.search.utils.SearchUtils; -import com.linkedin.metadata.systemmetadata.SystemMetadataService; -import com.linkedin.metadata.timeseries.TimeseriesAspectService; -import com.linkedin.metadata.timeseries.transformer.TimeseriesAspectTransformer; -import com.linkedin.metadata.utils.EntityKeyUtils; -import com.linkedin.metadata.utils.GenericRecordUtils; -import com.linkedin.mxe.GenericAspect; +import com.linkedin.metadata.service.UpdateIndicesService; import com.linkedin.mxe.MetadataChangeLog; -import com.linkedin.mxe.SystemMetadata; -import com.linkedin.util.Pair; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; import javax.annotation.Nonnull; -import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Import; import org.springframework.stereotype.Component; import static com.linkedin.metadata.Constants.*; -import static com.linkedin.metadata.search.utils.QueryUtils.*; + // TODO: Backfill tests for this class in UpdateIndicesHookTest.java @Slf4j @@ -73,50 +24,13 @@ EntityRegistryFactory.class, SystemMetadataServiceFactory.class, SearchDocumentTransformerFactory.class}) public class UpdateIndicesHook implements MetadataChangeLogHook { - private static final Set UPDATE_CHANGE_TYPES = ImmutableSet.of( - ChangeType.UPSERT, - ChangeType.RESTATE, - ChangeType.PATCH); - private static final String DOWNSTREAM_OF = "DownstreamOf"; - - private final GraphService _graphService; - private final EntitySearchService _entitySearchService; - private final TimeseriesAspectService _timeseriesAspectService; - private final SystemMetadataService _systemMetadataService; - private final EntityRegistry _entityRegistry; - private final SearchDocumentTransformer _searchDocumentTransformer; + private final UpdateIndicesService _updateIndicesService; private final boolean _isEnabled; - @Value("${featureFlags.graphServiceDiffModeEnabled:true}") - private boolean _graphDiffMode; - @Value("${featureFlags.searchServiceDiffModeEnabled:true}") - private boolean _searchDiffMode; - - @VisibleForTesting - void setGraphDiffMode(boolean graphDiffMode) { - _graphDiffMode = graphDiffMode; - } - - @VisibleForTesting - void setSearchDiffMode(boolean searchDiffMode) { - _searchDiffMode = searchDiffMode; - } - - @Autowired public UpdateIndicesHook( - GraphService graphService, - EntitySearchService entitySearchService, - TimeseriesAspectService timeseriesAspectService, - SystemMetadataService systemMetadataService, - EntityRegistry entityRegistry, - SearchDocumentTransformer searchDocumentTransformer, + UpdateIndicesService updateIndicesService, @Nonnull @Value("${updateIndices.enabled:true}") Boolean isEnabled) { - _graphService = graphService; - _entitySearchService = entitySearchService; - _timeseriesAspectService = timeseriesAspectService; - _systemMetadataService = systemMetadataService; - _entityRegistry = entityRegistry; - _searchDocumentTransformer = searchDocumentTransformer; + _updateIndicesService = updateIndicesService; _isEnabled = isEnabled; } @@ -127,449 +41,14 @@ public boolean isEnabled() { @Override public void invoke(@Nonnull final MetadataChangeLog event) { - if (UPDATE_CHANGE_TYPES.contains(event.getChangeType())) { - handleUpdateChangeEvent(event); - } else if (event.getChangeType() == ChangeType.DELETE) { - handleDeleteChangeEvent(event); - } - } - - /** - * This very important method processes {@link MetadataChangeLog} events - * that represent changes to the Metadata Graph. - * - * In particular, it handles updating the Search, Graph, Timeseries, and - * System Metadata stores in response to a given change type to reflect - * the changes present in the new aspect. - * - * @param event the change event to be processed. - */ - private void handleUpdateChangeEvent(@Nonnull final MetadataChangeLog event) { - - final EntitySpec entitySpec = getEventEntitySpec(event); - final Urn urn = EntityKeyUtils.getUrnFromLog(event, entitySpec.getKeyAspectSpec()); - - if (!event.hasAspectName() || !event.hasAspect()) { - log.error("Aspect or aspect name is missing. Skipping aspect processing..."); - return; - } - - AspectSpec aspectSpec = entitySpec.getAspectSpec(event.getAspectName()); - if (aspectSpec == null) { - throw new RuntimeException( - String.format("Failed to retrieve Aspect Spec for entity with name %s, aspect with name %s. Cannot update indices for MCL.", - event.getEntityType(), - event.getAspectName())); - } - - RecordTemplate aspect = GenericRecordUtils.deserializeAspect( - event.getAspect().getValue(), - event.getAspect().getContentType(), - aspectSpec); - GenericAspect previousAspectValue = event.getPreviousAspectValue(); - RecordTemplate previousAspect = previousAspectValue != null - ? GenericRecordUtils.deserializeAspect(previousAspectValue.getValue(), previousAspectValue.getContentType(), aspectSpec) - : null; - - // Step 0. If the aspect is timeseries, add to its timeseries index. - if (aspectSpec.isTimeseries()) { - updateTimeseriesFields(event.getEntityType(), event.getAspectName(), urn, aspect, aspectSpec, - event.getSystemMetadata()); - } else { - // Inject into the System Metadata Index when an aspect is non-timeseries only. - // TODO: Verify whether timeseries aspects can be dropped into System Metadata as well - // without impacting rollbacks. - updateSystemMetadata(event.getSystemMetadata(), urn, aspectSpec, aspect); - } - - // Step 1. For all aspects, attempt to update Search - updateSearchService(entitySpec.getName(), urn, aspectSpec, aspect, event.getSystemMetadata(), previousAspect); - - // Step 2. For all aspects, attempt to update Graph - SystemMetadata systemMetadata = event.getSystemMetadata(); - if (_graphDiffMode && !(_graphService instanceof DgraphGraphService) - && (systemMetadata == null || systemMetadata.getProperties() == null - || !Boolean.parseBoolean(systemMetadata.getProperties().get(FORCE_INDEXING_KEY)))) { - updateGraphServiceDiff(urn, aspectSpec, previousAspect, aspect, event); - } else { - updateGraphService(urn, aspectSpec, aspect, event); - } - } - - /** - * This very important method processes {@link MetadataChangeLog} deletion events - * to cleanup the Metadata Graph when an aspect or entity is removed. - * - * In particular, it handles updating the Search, Graph, Timeseries, and - * System Metadata stores to reflect the deletion of a particular aspect. - * - * Note that if an entity's key aspect is deleted, the entire entity will be purged - * from search, graph, timeseries, etc. - * - * @param event the change event to be processed. - */ - private void handleDeleteChangeEvent(@Nonnull final MetadataChangeLog event) { - - final EntitySpec entitySpec = getEventEntitySpec(event); - final Urn urn = EntityKeyUtils.getUrnFromLog(event, entitySpec.getKeyAspectSpec()); - - if (!event.hasAspectName() || !event.hasPreviousAspectValue()) { - log.error("Previous aspect or aspect name is missing. Skipping aspect processing..."); - return; - } - - AspectSpec aspectSpec = entitySpec.getAspectSpec(event.getAspectName()); - if (aspectSpec == null) { - throw new RuntimeException( - String.format("Failed to retrieve Aspect Spec for entity with name %s, aspect with name %s. Cannot update indices for MCL.", - event.getEntityType(), - event.getAspectName())); - } - - RecordTemplate aspect = GenericRecordUtils.deserializeAspect(event.getPreviousAspectValue().getValue(), - event.getPreviousAspectValue().getContentType(), aspectSpec); - Boolean isDeletingKey = event.getAspectName().equals(entitySpec.getKeyAspectName()); - - if (!aspectSpec.isTimeseries()) { - deleteSystemMetadata(urn, aspectSpec, isDeletingKey); - deleteGraphData(urn, aspectSpec, aspect, isDeletingKey, event); - deleteSearchData(urn, entitySpec.getName(), aspectSpec, aspect, isDeletingKey); - } - } - - // TODO: remove this method once we implement sourceOverride when creating graph edges - private void updateFineGrainedEdgesAndRelationships( - RecordTemplate aspect, - List edgesToAdd, - HashMap> urnToRelationshipTypesBeingAdded - ) { - UpstreamLineage upstreamLineage = new UpstreamLineage(aspect.data()); - if (upstreamLineage.getFineGrainedLineages() != null) { - for (FineGrainedLineage fineGrainedLineage : upstreamLineage.getFineGrainedLineages()) { - if (!fineGrainedLineage.hasDownstreams() || !fineGrainedLineage.hasUpstreams()) { - break; - } - // for every downstream, create an edge with each of the upstreams - for (Urn downstream : fineGrainedLineage.getDownstreams()) { - for (Urn upstream : fineGrainedLineage.getUpstreams()) { - // TODO: add edges uniformly across aspects - edgesToAdd.add(new Edge(downstream, upstream, DOWNSTREAM_OF, null, null, null, null, null)); - Set relationshipTypes = urnToRelationshipTypesBeingAdded.getOrDefault(downstream, new HashSet<>()); - relationshipTypes.add(DOWNSTREAM_OF); - urnToRelationshipTypesBeingAdded.put(downstream, relationshipTypes); - } - } - } - } - } - - private Urn generateSchemaFieldUrn(@Nonnull final String resourceUrn, @Nonnull final String fieldPath) { - // we rely on schemaField fieldPaths to be encoded since we do that with fineGrainedLineage on the ingestion side - final String encodedFieldPath = fieldPath.replaceAll("\\(", "%28").replaceAll("\\)", "%29").replaceAll(",", "%2C"); - final SchemaFieldKey key = new SchemaFieldKey().setParent(UrnUtils.getUrn(resourceUrn)).setFieldPath(encodedFieldPath); - return EntityKeyUtils.convertEntityKeyToUrn(key, Constants.SCHEMA_FIELD_ENTITY_NAME); - } - - // TODO: remove this method once we implement sourceOverride and update inputFields aspect - private void updateInputFieldEdgesAndRelationships( - @Nonnull final Urn urn, - @Nonnull final InputFields inputFields, - @Nonnull final List edgesToAdd, - @Nonnull final HashMap> urnToRelationshipTypesBeingAdded - ) { - if (inputFields.hasFields()) { - for (final InputField field : inputFields.getFields()) { - if (field.hasSchemaFieldUrn() && field.hasSchemaField() && field.getSchemaField().hasFieldPath()) { - final Urn sourceFieldUrn = generateSchemaFieldUrn(urn.toString(), field.getSchemaField().getFieldPath()); - // TODO: add edges uniformly across aspects - edgesToAdd.add(new Edge(sourceFieldUrn, field.getSchemaFieldUrn(), DOWNSTREAM_OF, null, null, null, null, null)); - final Set relationshipTypes = urnToRelationshipTypesBeingAdded.getOrDefault(sourceFieldUrn, new HashSet<>()); - relationshipTypes.add(DOWNSTREAM_OF); - urnToRelationshipTypesBeingAdded.put(sourceFieldUrn, relationshipTypes); - } - } - } - } - - private Pair, HashMap>> getEdgesAndRelationshipTypesFromAspect( - @Nonnull final Urn urn, - @Nonnull final AspectSpec aspectSpec, - @Nonnull final RecordTemplate aspect, - @Nonnull final MetadataChangeLog event, - final boolean isNewAspectVersion - ) { - final List edgesToAdd = new ArrayList<>(); - final HashMap> urnToRelationshipTypesBeingAdded = new HashMap<>(); - - // we need to manually set schemaField <-> schemaField edges for fineGrainedLineage and inputFields - // since @Relationship only links between the parent entity urn and something else. - if (aspectSpec.getName().equals(Constants.UPSTREAM_LINEAGE_ASPECT_NAME)) { - updateFineGrainedEdgesAndRelationships(aspect, edgesToAdd, urnToRelationshipTypesBeingAdded); - } - if (aspectSpec.getName().equals(Constants.INPUT_FIELDS_ASPECT_NAME)) { - final InputFields inputFields = new InputFields(aspect.data()); - updateInputFieldEdgesAndRelationships(urn, inputFields, edgesToAdd, urnToRelationshipTypesBeingAdded); - } - - Map> extractedFields = - FieldExtractor.extractFields(aspect, aspectSpec.getRelationshipFieldSpecs()); - - for (Map.Entry> entry : extractedFields.entrySet()) { - Set relationshipTypes = urnToRelationshipTypesBeingAdded.getOrDefault(urn, new HashSet<>()); - relationshipTypes.add(entry.getKey().getRelationshipName()); - urnToRelationshipTypesBeingAdded.put(urn, relationshipTypes); - final List newEdges = GraphIndexUtils.extractGraphEdges(entry, aspect, urn, event, isNewAspectVersion); - edgesToAdd.addAll(newEdges); - } - return Pair.of(edgesToAdd, urnToRelationshipTypesBeingAdded); - } - - /** - * Process snapshot and update graph index - */ - private void updateGraphService( - @Nonnull final Urn urn, - @Nonnull final AspectSpec aspectSpec, - @Nonnull final RecordTemplate aspect, - @Nonnull final MetadataChangeLog event - ) { - Pair, HashMap>> edgeAndRelationTypes = - getEdgesAndRelationshipTypesFromAspect(urn, aspectSpec, aspect, event, true); - - final List edgesToAdd = edgeAndRelationTypes.getFirst(); - final HashMap> urnToRelationshipTypesBeingAdded = edgeAndRelationTypes.getSecond(); - - log.debug("Here's the relationship types found {}", urnToRelationshipTypesBeingAdded); - if (urnToRelationshipTypesBeingAdded.size() > 0) { - for (Map.Entry> entry : urnToRelationshipTypesBeingAdded.entrySet()) { - _graphService.removeEdgesFromNode(entry.getKey(), new ArrayList<>(entry.getValue()), - newRelationshipFilter(new Filter().setOr(new ConjunctiveCriterionArray()), RelationshipDirection.OUTGOING)); - } - edgesToAdd.forEach(_graphService::addEdge); - } - } - - private void updateGraphServiceDiff( - @Nonnull final Urn urn, - @Nonnull final AspectSpec aspectSpec, - @Nullable final RecordTemplate oldAspect, - @Nonnull final RecordTemplate newAspect, - @Nonnull final MetadataChangeLog event - ) { - Pair, HashMap>> oldEdgeAndRelationTypes = null; - if (oldAspect != null) { - oldEdgeAndRelationTypes = getEdgesAndRelationshipTypesFromAspect(urn, aspectSpec, oldAspect, event, false); - } - - final List oldEdges = - oldEdgeAndRelationTypes != null ? oldEdgeAndRelationTypes.getFirst() : Collections.emptyList(); - final Set oldEdgeSet = new HashSet<>(oldEdges); - - Pair, HashMap>> newEdgeAndRelationTypes = - getEdgesAndRelationshipTypesFromAspect(urn, aspectSpec, newAspect, event, true); - - final List newEdges = newEdgeAndRelationTypes.getFirst(); - final Set newEdgeSet = new HashSet<>(newEdges); - - // Edges to add - final List additiveDifference = newEdgeSet.stream() - .filter(edge -> !oldEdgeSet.contains(edge)) - .collect(Collectors.toList()); - - // Edges to remove - final List subtractiveDifference = oldEdgeSet.stream() - .filter(edge -> !newEdgeSet.contains(edge)) - .collect(Collectors.toList()); - - // Edges to update - final List mergedEdges = getMergedEdges(oldEdgeSet, newEdgeSet); - - // Remove any old edges that no longer exist first - if (subtractiveDifference.size() > 0) { - log.debug("Removing edges: {}", subtractiveDifference); - subtractiveDifference.forEach(_graphService::removeEdge); - } - - // Then add new edges - if (additiveDifference.size() > 0) { - log.debug("Adding edges: {}", additiveDifference); - additiveDifference.forEach(_graphService::addEdge); - } - - // Then update existing edges - if (mergedEdges.size() > 0) { - log.debug("Updating edges: {}", mergedEdges); - mergedEdges.forEach(_graphService::upsertEdge); - } - } - - private static List getMergedEdges(final Set oldEdgeSet, final Set newEdgeSet) { - final Map oldEdgesMap = oldEdgeSet - .stream() - .map(edge -> Pair.of(edge.hashCode(), edge)) - .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond)); - - final List mergedEdges = new ArrayList<>(); - if (!oldEdgesMap.isEmpty()) { - for (com.linkedin.metadata.graph.Edge newEdge : newEdgeSet) { - if (oldEdgesMap.containsKey(newEdge.hashCode())) { - final com.linkedin.metadata.graph.Edge oldEdge = oldEdgesMap.get(newEdge.hashCode()); - final com.linkedin.metadata.graph.Edge mergedEdge = GraphIndexUtils.mergeEdges(oldEdge, newEdge); - mergedEdges.add(mergedEdge); - } - } - } - - return mergedEdges; - } - - /** - * Process snapshot and update search index - */ - private void updateSearchService(String entityName, Urn urn, AspectSpec aspectSpec, RecordTemplate aspect, - @Nullable SystemMetadata systemMetadata, @Nullable RecordTemplate previousAspect) { - Optional searchDocument; - Optional previousSearchDocument = Optional.empty(); - try { - searchDocument = _searchDocumentTransformer.transformAspect(urn, aspect, aspectSpec, false); - } catch (Exception e) { - log.error("Error in getting documents from aspect: {} for aspect {}", e, aspectSpec.getName()); - return; - } - - if (!searchDocument.isPresent()) { - return; - } - - Optional docId = SearchUtils.getDocId(urn); - - if (!docId.isPresent()) { - return; - } - - String searchDocumentValue = searchDocument.get(); - if (_searchDiffMode && (systemMetadata == null || systemMetadata.getProperties() == null - || !Boolean.parseBoolean(systemMetadata.getProperties().get(FORCE_INDEXING_KEY)))) { - if (previousAspect != null) { - try { - previousSearchDocument = _searchDocumentTransformer.transformAspect(urn, previousAspect, aspectSpec, false); - } catch (Exception e) { - log.error( - "Error in getting documents from previous aspect state: {} for aspect {}, continuing without diffing.", e, - aspectSpec.getName()); - } - } - - if (previousSearchDocument.isPresent()) { - String previousSearchDocumentValue = previousSearchDocument.get(); - if (searchDocumentValue.equals(previousSearchDocumentValue)) { - // No changes to search document, skip writing no-op update + if (event.getSystemMetadata() != null) { + if (event.getSystemMetadata().getProperties() != null) { + if (UI_SOURCE.equals(event.getSystemMetadata().getProperties().get(APP_SOURCE))) { + // If coming from the UI, we pre-process the Update Indices hook as a fast path to avoid Kafka lag return; } } } - - _entitySearchService.upsertDocument(entityName, searchDocument.get(), docId.get()); - } - - /** - * Process snapshot and update time-series index - */ - private void updateTimeseriesFields(String entityType, String aspectName, Urn urn, RecordTemplate aspect, - AspectSpec aspectSpec, SystemMetadata systemMetadata) { - Map documents; - try { - documents = TimeseriesAspectTransformer.transform(urn, aspect, aspectSpec, systemMetadata); - } catch (JsonProcessingException e) { - log.error("Failed to generate timeseries document from aspect: {}", e.toString()); - return; - } - documents.entrySet().forEach(document -> { - _timeseriesAspectService.upsertDocument(entityType, aspectName, document.getKey(), document.getValue()); - }); - } - - private void updateSystemMetadata(SystemMetadata systemMetadata, Urn urn, AspectSpec aspectSpec, RecordTemplate aspect) { - _systemMetadataService.insert(systemMetadata, urn.toString(), aspectSpec.getName()); - - // If processing status aspect update all aspects for this urn to removed - if (aspectSpec.getName().equals(Constants.STATUS_ASPECT_NAME)) { - _systemMetadataService.setDocStatus(urn.toString(), ((Status) aspect).isRemoved()); - } - } - - private void deleteSystemMetadata(Urn urn, AspectSpec aspectSpec, Boolean isKeyAspect) { - if (isKeyAspect) { - // Delete all aspects - log.debug(String.format("Deleting all system metadata for urn: %s", urn)); - _systemMetadataService.deleteUrn(urn.toString()); - } else { - // Delete all aspects from system metadata service - log.debug(String.format("Deleting system metadata for urn: %s, aspect: %s", urn, aspectSpec.getName())); - _systemMetadataService.deleteAspect(urn.toString(), aspectSpec.getName()); - } - } - - private void deleteGraphData( - @Nonnull final Urn urn, - @Nonnull final AspectSpec aspectSpec, - @Nonnull final RecordTemplate aspect, - @Nonnull final Boolean isKeyAspect, - @Nonnull final MetadataChangeLog event - ) { - if (isKeyAspect) { - _graphService.removeNode(urn); - return; - } - - Pair, HashMap>> edgeAndRelationTypes = - getEdgesAndRelationshipTypesFromAspect(urn, aspectSpec, aspect, event, true); - - final HashMap> urnToRelationshipTypesBeingAdded = edgeAndRelationTypes.getSecond(); - if (urnToRelationshipTypesBeingAdded.size() > 0) { - for (Map.Entry> entry : urnToRelationshipTypesBeingAdded.entrySet()) { - _graphService.removeEdgesFromNode(entry.getKey(), new ArrayList<>(entry.getValue()), - createRelationshipFilter(new Filter().setOr(new ConjunctiveCriterionArray()), RelationshipDirection.OUTGOING)); - } - } - } - - private void deleteSearchData(Urn urn, String entityName, AspectSpec aspectSpec, RecordTemplate aspect, Boolean isKeyAspect) { - String docId; - try { - docId = URLEncoder.encode(urn.toString(), "UTF-8"); - } catch (UnsupportedEncodingException e) { - log.error("Failed to encode the urn with error: {}", e.toString()); - return; - } - - if (isKeyAspect) { - _entitySearchService.deleteDocument(entityName, docId); - return; - } - - Optional searchDocument; - try { - searchDocument = _searchDocumentTransformer.transformAspect(urn, aspect, aspectSpec, true); // TODO - } catch (Exception e) { - log.error("Error in getting documents from aspect: {} for aspect {}", e, aspectSpec.getName()); - return; - } - - if (!searchDocument.isPresent()) { - return; - } - - _entitySearchService.upsertDocument(entityName, searchDocument.get(), docId); - } - - private EntitySpec getEventEntitySpec(@Nonnull final MetadataChangeLog event) { - try { - return _entityRegistry.getEntitySpec(event.getEntityType()); - } catch (IllegalArgumentException e) { - throw new RuntimeException( - String.format("Failed to retrieve Entity Spec for entity with name %s. Cannot update indices for MCL.", - event.getEntityType())); - } + _updateIndicesService.handleChangeEvent(event); } } diff --git a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/GraphIndexUtilsTest.java b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/GraphIndexUtilsTest.java index 1dffe21d2e790..6428a19c3909b 100644 --- a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/GraphIndexUtilsTest.java +++ b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/GraphIndexUtilsTest.java @@ -12,6 +12,7 @@ import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; import com.linkedin.metadata.graph.Edge; +import com.linkedin.metadata.graph.GraphIndexUtils; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.RelationshipFieldSpec; @@ -29,7 +30,7 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import static com.linkedin.metadata.kafka.hook.GraphIndexUtils.*; +import static com.linkedin.metadata.graph.GraphIndexUtils.*; import static org.testng.Assert.*; diff --git a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHookTest.java b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHookTest.java index 87172518cc2cc..798831e90452c 100644 --- a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHookTest.java +++ b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHookTest.java @@ -10,6 +10,7 @@ import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.schema.RecordDataSchema; import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.StringMap; import com.linkedin.dataset.DatasetLineageType; import com.linkedin.dataset.FineGrainedLineage; import com.linkedin.dataset.FineGrainedLineageArray; @@ -35,10 +36,12 @@ import com.linkedin.metadata.query.filter.RelationshipDirection; import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.search.transformer.SearchDocumentTransformer; +import com.linkedin.metadata.service.UpdateIndicesService; import com.linkedin.metadata.systemmetadata.SystemMetadataService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeLog; +import com.linkedin.mxe.SystemMetadata; import com.linkedin.schema.SchemaField; import java.net.URISyntaxException; import java.net.URLEncoder; @@ -50,7 +53,7 @@ import java.util.ArrayList; import java.util.Collections; -import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME; +import static com.linkedin.metadata.Constants.*; import static com.linkedin.metadata.kafka.hook.MCLProcessingTestDataGenerator.*; import static com.linkedin.metadata.search.utils.QueryUtils.newRelationshipFilter; @@ -80,6 +83,7 @@ public class UpdateIndicesHookTest { private DataHubUpgradeKafkaListener _mockDataHubUpgradeKafkaListener; private ConfigurationProvider _mockConfigurationProvider; private Urn _actorUrn; + private UpdateIndicesService _updateIndicesService; @BeforeMethod public void setupTest() { @@ -97,20 +101,23 @@ public void setupTest() { SystemUpdateConfiguration systemUpdateConfiguration = new SystemUpdateConfiguration(); systemUpdateConfiguration.setWaitForSystemUpdate(false); Mockito.when(_mockConfigurationProvider.getElasticSearch()).thenReturn(elasticSearchConfiguration); - _updateIndicesHook = new UpdateIndicesHook( + _updateIndicesService = new UpdateIndicesService( _mockGraphService, _mockEntitySearchService, _mockTimeseriesAspectService, _mockSystemMetadataService, registry, - _searchDocumentTransformer, + _searchDocumentTransformer + ); + _updateIndicesHook = new UpdateIndicesHook( + _updateIndicesService, true ); } @Test public void testFineGrainedLineageEdgesAreAdded() throws Exception { - _updateIndicesHook.setGraphDiffMode(false); + _updateIndicesService.setGraphDiffMode(false); Urn upstreamUrn = UrnUtils.getUrn("urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD),foo_info)"); Urn downstreamUrn = UrnUtils.getUrn("urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD),field_foo)"); MetadataChangeLog event = createUpstreamLineageMCL(upstreamUrn, downstreamUrn); @@ -127,7 +134,7 @@ public void testFineGrainedLineageEdgesAreAdded() throws Exception { @Test public void testFineGrainedLineageEdgesAreAddedRestate() throws Exception { - _updateIndicesHook.setGraphDiffMode(false); + _updateIndicesService.setGraphDiffMode(false); Urn upstreamUrn = UrnUtils.getUrn("urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD),foo_info)"); Urn downstreamUrn = UrnUtils.getUrn("urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD),field_foo)"); MetadataChangeLog event = createUpstreamLineageMCL(upstreamUrn, downstreamUrn, ChangeType.RESTATE); @@ -152,15 +159,15 @@ public void testInputFieldsEdgesAreAdded() throws Exception { String downstreamFieldPath = "users.count"; MetadataChangeLog event = createInputFieldsMCL(upstreamUrn, downstreamFieldPath); EntityRegistry mockEntityRegistry = createMockEntityRegistry(); - _updateIndicesHook = new UpdateIndicesHook( + _updateIndicesService = new UpdateIndicesService( _mockGraphService, _mockEntitySearchService, _mockTimeseriesAspectService, _mockSystemMetadataService, mockEntityRegistry, - _searchDocumentTransformer, - true + _searchDocumentTransformer ); + _updateIndicesHook = new UpdateIndicesHook(_updateIndicesService, true); _updateIndicesHook.invoke(event); @@ -178,8 +185,8 @@ public void testInputFieldsEdgesAreAdded() throws Exception { @Test public void testMCLProcessExhaustive() throws URISyntaxException { - _updateIndicesHook.setGraphDiffMode(true); - _updateIndicesHook.setSearchDiffMode(true); + _updateIndicesService.setGraphDiffMode(true); + _updateIndicesService.setSearchDiffMode(true); /* * newLineage */ @@ -349,6 +356,18 @@ public void testMCLProcessExhaustive() throws URISyntaxException { .upsertDocument(Mockito.any(), Mockito.any(), Mockito.any()); } + @Test + public void testMCLUIPreProcessed() throws Exception { + _updateIndicesService.setGraphDiffMode(true); + _updateIndicesService.setSearchDiffMode(true); + Urn upstreamUrn = UrnUtils.getUrn("urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD),foo_info)"); + Urn downstreamUrn = UrnUtils.getUrn("urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD),field_foo)"); + + MetadataChangeLog changeLog = createUpstreamLineageMCLUIPreProcessed(upstreamUrn, downstreamUrn, ChangeType.UPSERT); + _updateIndicesHook.invoke(changeLog); + Mockito.verifyNoInteractions(_mockEntitySearchService, _mockGraphService, _mockTimeseriesAspectService, _mockSystemMetadataService); + } + private EntityRegistry createMockEntityRegistry() { // need to mock this registry instead of using test-entity-registry.yml because inputFields does not work due to a known bug EntityRegistry mockEntityRegistry = Mockito.mock(EntityRegistry.class); @@ -410,6 +429,15 @@ private MetadataChangeLog createUpstreamLineageMCL(Urn upstreamUrn, Urn downstre return event; } + private MetadataChangeLog createUpstreamLineageMCLUIPreProcessed(Urn upstreamUrn, Urn downstreamUrn, ChangeType changeType) throws Exception { + final MetadataChangeLog metadataChangeLog = createUpstreamLineageMCL(upstreamUrn, downstreamUrn, changeType); + final StringMap properties = new StringMap(); + properties.put(APP_SOURCE, UI_SOURCE); + final SystemMetadata systemMetadata = new SystemMetadata().setProperties(properties); + metadataChangeLog.setSystemMetadata(systemMetadata); + return metadataChangeLog; + } + private MetadataChangeLog createInputFieldsMCL(Urn upstreamUrn, String downstreamFieldPath) throws Exception { MetadataChangeLog event = new MetadataChangeLog(); event.setEntityType(Constants.CHART_ENTITY_NAME); diff --git a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTestConfiguration.java b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTestConfiguration.java index 8196ed43dfc0c..206b15ff61cca 100644 --- a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTestConfiguration.java +++ b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTestConfiguration.java @@ -21,7 +21,8 @@ @Configuration @ComponentScan(basePackages = { - "com.linkedin.metadata.kafka" + "com.linkedin.metadata.kafka", + "com.linkedin.gms.factory.entity.update.indices" }) public class MCLSpringTestConfiguration { diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityServiceFactory.java index a543cca27acff..6bb86b01604c1 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityServiceFactory.java @@ -1,5 +1,6 @@ package com.linkedin.gms.factory.entity; +import com.linkedin.datahub.graphql.featureflags.FeatureFlags; import com.linkedin.gms.factory.common.TopicConventionFactory; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.metadata.dao.producer.KafkaEventProducer; @@ -7,6 +8,7 @@ import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.service.UpdateIndicesService; import com.linkedin.mxe.TopicConvention; import org.apache.avro.generic.IndexedRecord; import org.apache.kafka.clients.producer.Producer; @@ -31,9 +33,12 @@ protected EntityService createInstance( KafkaHealthChecker kafkaHealthChecker, @Qualifier("entityAspectDao") AspectDao aspectDao, EntityRegistry entityRegistry, - ConfigurationProvider configurationProvider) { + ConfigurationProvider configurationProvider, + UpdateIndicesService updateIndicesService) { final KafkaEventProducer eventProducer = new KafkaEventProducer(producer, convention, kafkaHealthChecker); - return new EntityService(aspectDao, eventProducer, entityRegistry, configurationProvider.getFeatureFlags().isAlwaysEmitChangeLog()); + FeatureFlags featureFlags = configurationProvider.getFeatureFlags(); + return new EntityService(aspectDao, eventProducer, entityRegistry, + featureFlags.isAlwaysEmitChangeLog(), updateIndicesService, featureFlags.getPreProcessHooks()); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/update/indices/UpdateIndicesServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/update/indices/UpdateIndicesServiceFactory.java new file mode 100644 index 0000000000000..f86f6bf7d0877 --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/update/indices/UpdateIndicesServiceFactory.java @@ -0,0 +1,24 @@ +package com.linkedin.gms.factory.entity.update.indices; + +import com.linkedin.metadata.graph.GraphService; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.search.transformer.SearchDocumentTransformer; +import com.linkedin.metadata.service.UpdateIndicesService; +import com.linkedin.metadata.systemmetadata.SystemMetadataService; +import com.linkedin.metadata.timeseries.TimeseriesAspectService; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + + +@Configuration +public class UpdateIndicesServiceFactory { + + @Bean + public UpdateIndicesService updateIndicesService(GraphService graphService, EntitySearchService entitySearchService, + TimeseriesAspectService timeseriesAspectService, SystemMetadataService systemMetadataService, + EntityRegistry entityRegistry, SearchDocumentTransformer searchDocumentTransformer) { + return new UpdateIndicesService(graphService, entitySearchService, timeseriesAspectService, + systemMetadataService, entityRegistry, searchDocumentTransformer); + } +} diff --git a/metadata-service/factories/src/main/resources/application.yml b/metadata-service/factories/src/main/resources/application.yml index 80c57e334fd53..adbb8483f345b 100644 --- a/metadata-service/factories/src/main/resources/application.yml +++ b/metadata-service/factories/src/main/resources/application.yml @@ -283,6 +283,8 @@ featureFlags: alwaysEmitChangeLog: ${ALWAYS_EMIT_CHANGE_LOG:false} # Enables always emitting a MCL even when no changes are detected. Used for Time Based Lineage when no changes occur. searchServiceDiffModeEnabled: ${SEARCH_SERVICE_DIFF_MODE_ENABLED:true} # Enables diff mode for search document writes, reduces amount of writes to ElasticSearch documents for no-ops readOnlyModeEnabled: ${READ_ONLY_MODE_ENABLED:false} # Enables read only mode for an instance. Right now this only affects ability to edit user profile image URL but can be extended + preProcessHooks: + uiEnabled: ${PRE_PROCESS_HOOKS_UI_ENABLED:true} # Circumvents Kafka for processing index updates for UI changes sourced from GraphQL to avoid processing delays entityChangeEvents: enabled: ${ENABLE_ENTITY_CHANGE_EVENTS_HOOK:true} diff --git a/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java b/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java index 64e74d0c42c9e..2dff636de2f27 100644 --- a/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java +++ b/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java @@ -7,9 +7,11 @@ import com.datahub.authorization.AuthorizationResult; import com.datahub.authorization.AuthorizerChain; import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.service.UpdateIndicesService; import io.datahubproject.openapi.dto.UpsertAspectRequest; import io.datahubproject.openapi.entities.EntitiesController; import io.datahubproject.openapi.generated.AuditStamp; @@ -41,7 +43,7 @@ import static com.linkedin.metadata.Constants.*; import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; public class EntitiesControllerTest { @@ -59,7 +61,11 @@ public void setup() EntityRegistry mockEntityRegistry = new MockEntityRegistry(); AspectDao aspectDao = Mockito.mock(AspectDao.class); EventProducer mockEntityEventProducer = Mockito.mock(EventProducer.class); - MockEntityService mockEntityService = new MockEntityService(aspectDao, mockEntityEventProducer, mockEntityRegistry); + UpdateIndicesService mockUpdateIndicesService = mock(UpdateIndicesService.class); + PreProcessHooks preProcessHooks = new PreProcessHooks(); + preProcessHooks.setUiEnabled(true); + MockEntityService mockEntityService = new MockEntityService(aspectDao, mockEntityEventProducer, mockEntityRegistry, + mockUpdateIndicesService, preProcessHooks); AuthorizerChain authorizerChain = Mockito.mock(AuthorizerChain.class); _entitiesController = new EntitiesController(mockEntityService, new ObjectMapper(), authorizerChain); Authentication authentication = Mockito.mock(Authentication.class); diff --git a/metadata-service/openapi-servlet/src/test/java/mock/MockEntityService.java b/metadata-service/openapi-servlet/src/test/java/mock/MockEntityService.java index bf32a4a73c224..9838b00abd191 100644 --- a/metadata-service/openapi-servlet/src/test/java/mock/MockEntityService.java +++ b/metadata-service/openapi-servlet/src/test/java/mock/MockEntityService.java @@ -19,6 +19,7 @@ import com.linkedin.entity.AspectType; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.metadata.aspect.VersionedAspect; +import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.ListResult; @@ -28,6 +29,7 @@ import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.query.ListUrnsResult; import com.linkedin.metadata.run.AspectRowSummary; +import com.linkedin.metadata.service.UpdateIndicesService; import com.linkedin.mxe.SystemMetadata; import com.linkedin.schema.ForeignKeyConstraint; import com.linkedin.schema.ForeignKeyConstraintArray; @@ -55,8 +57,9 @@ public class MockEntityService extends EntityService { - public MockEntityService(@Nonnull AspectDao aspectDao, @Nonnull EventProducer producer, @Nonnull EntityRegistry entityRegistry) { - super(aspectDao, producer, entityRegistry, true); + public MockEntityService(@Nonnull AspectDao aspectDao, @Nonnull EventProducer producer, @Nonnull EntityRegistry entityRegistry, @Nonnull + UpdateIndicesService updateIndicesService, PreProcessHooks preProcessHooks) { + super(aspectDao, producer, entityRegistry, true, updateIndicesService, preProcessHooks); } @Override diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/utils/PegasusUtils.java b/metadata-utils/src/main/java/com/linkedin/metadata/utils/PegasusUtils.java index a680a845fb2f5..9794d101ecda9 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/utils/PegasusUtils.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/utils/PegasusUtils.java @@ -2,13 +2,19 @@ import com.datahub.util.RecordUtils; import com.datahub.util.exception.ModelConversionException; +import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; import com.linkedin.data.schema.NamedDataSchema; import com.linkedin.data.schema.RecordDataSchema; import com.linkedin.data.template.RecordTemplate; +import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.models.annotation.AspectAnnotation; import com.linkedin.metadata.models.annotation.EntityAnnotation; +import com.linkedin.mxe.MetadataChangeLog; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.mxe.SystemMetadata; +import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; @@ -60,4 +66,33 @@ public static Class getDataTemplateClassFromSchema(final NamedD public static String urnToEntityName(final Urn urn) { return urn.getEntityType(); } + + public static MetadataChangeLog constructMCL(@Nullable MetadataChangeProposal base, String entityName, Urn urn, ChangeType changeType, + String aspectName, AuditStamp auditStamp, RecordTemplate newAspectValue, SystemMetadata newSystemMetadata, + RecordTemplate oldAspectValue, SystemMetadata oldSystemMetadata) { + final MetadataChangeLog metadataChangeLog; + if (base != null) { + metadataChangeLog = new MetadataChangeLog(new DataMap(base.data())); + } else { + metadataChangeLog = new MetadataChangeLog(); + } + metadataChangeLog.setEntityType(entityName); + metadataChangeLog.setEntityUrn(urn); + metadataChangeLog.setChangeType(changeType); + metadataChangeLog.setAspectName(aspectName); + metadataChangeLog.setCreated(auditStamp); + if (newAspectValue != null) { + metadataChangeLog.setAspect(GenericRecordUtils.serializeAspect(newAspectValue)); + } + if (newSystemMetadata != null) { + metadataChangeLog.setSystemMetadata(newSystemMetadata); + } + if (oldAspectValue != null) { + metadataChangeLog.setPreviousAspectValue(GenericRecordUtils.serializeAspect(oldAspectValue)); + } + if (oldSystemMetadata != null) { + metadataChangeLog.setPreviousSystemMetadata(oldSystemMetadata); + } + return metadataChangeLog; + } } From a3ceee46d062caeadab2c0427b816b9bf0540f68 Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Thu, 15 Jun 2023 15:10:21 +0200 Subject: [PATCH 022/222] fix(ingest/druid) Handling gracefully if no table returned in a schema (#8203) --- .../src/datahub/ingestion/source/sql/druid.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/druid.py b/metadata-ingestion/src/datahub/ingestion/source/sql/druid.py index ad52707ec5500..1dfa44f549135 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/druid.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/druid.py @@ -1,6 +1,8 @@ # This import verifies that the dependencies are available. import pydruid # noqa: F401 from pydantic.fields import Field +from pydruid.db.sqlalchemy import DruidDialect +from sqlalchemy.exc import ResourceClosedError from datahub.configuration.common import AllowDenyPattern from datahub.ingestion.api.decorators import ( @@ -14,12 +16,25 @@ from datahub.ingestion.source.sql.sql_common import SQLAlchemySource from datahub.ingestion.source.sql.sql_config import BasicSQLAlchemyConfig +get_table_names_source = DruidDialect.get_table_names + + +def get_table_names(self, connection, schema=None, **kwargs): + try: + return get_table_names_source(self, connection, schema=schema, **kwargs) + # Druid throws ResourceClosedError when there is no table in the schema + except ResourceClosedError: + return [] + + +DruidDialect.get_table_names = get_table_names + class DruidConfig(BasicSQLAlchemyConfig): # defaults scheme = "druid" schema_pattern: AllowDenyPattern = Field( - default=AllowDenyPattern(deny=["^(lookup|sys).*"]), + default=AllowDenyPattern(deny=["^(lookup|sysgit|view).*"]), description="regex patterns for schemas to filter in ingestion.", ) From 8f9a23fb2e129cace52e7adc42978120b8e6103e Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Thu, 15 Jun 2023 12:30:04 -0500 Subject: [PATCH 023/222] fix(kafka-setup): bump kafka version (#8245) --- docker/kafka-setup/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/kafka-setup/Dockerfile b/docker/kafka-setup/Dockerfile index d2e0eeb727bc9..e379ae4385246 100644 --- a/docker/kafka-setup/Dockerfile +++ b/docker/kafka-setup/Dockerfile @@ -10,7 +10,7 @@ RUN rm /usr/share/java/cp-base-new/snakeyaml-*.jar \ # Based on https://github.com/blacktop's alpine kafka build FROM python:3-alpine -ENV KAFKA_VERSION 3.4.0 +ENV KAFKA_VERSION 3.4.1 ENV SCALA_VERSION 2.13 # Set the classpath for JARs required by `cub` From e7e07a73b4d3194d377909a45e450288c8b6c69a Mon Sep 17 00:00:00 2001 From: mohdsiddique Date: Fri, 16 Jun 2023 02:34:40 +0530 Subject: [PATCH 024/222] feat(ingestion/powerbi): Ingest datasets not used in PowerBI visualization(tiles/pages) (#8212) Co-authored-by: MohdSiddiqueBagwan --- .../ingestion/source/powerbi/config.py | 7 +- .../source/powerbi/m_query/parser.py | 2 +- .../ingestion/source/powerbi/powerbi.py | 32 ++++- .../powerbi/rest_api_wrapper/data_classes.py | 1 + .../powerbi/rest_api_wrapper/powerbi_api.py | 27 ++++ .../golden_test_independent_datasets.json | 73 +++++++++++ .../tests/integration/powerbi/test_powerbi.py | 116 ++++++++++++++++++ 7 files changed, 250 insertions(+), 8 deletions(-) create mode 100644 metadata-ingestion/tests/integration/powerbi/golden_test_independent_datasets.json diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py index dc92b55a9d93d..31d067f984d2d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py @@ -234,7 +234,7 @@ class PowerBiDashboardSourceConfig( hidden_from_docs=True, ) - # Organisation Identifier + # Organization Identifier tenant_id: str = pydantic.Field(description="PowerBI tenant identifier") # PowerBi workspace identifier workspace_id: Optional[str] = pydantic.Field( @@ -371,6 +371,11 @@ class PowerBiDashboardSourceConfig( description="Retrieve metadata using PowerBI Admin API only. If this is enabled, then Report Pages will not " "be extracted. Admin API access is required if this setting is enabled", ) + # Extract independent datasets + extract_independent_datasets: bool = pydantic.Field( + default=False, + description="Whether to extract datasets not used in any PowerBI visualization", + ) platform_instance: Optional[str] = pydantic.Field( default=None, diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/parser.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/parser.py index 5784d934b8b16..83106c04529d1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/parser.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/parser.py @@ -74,7 +74,7 @@ def get_upstream_tables( message = "Failed to parse m-query expression" reporter.report_warning(table.full_name, message) - logger.info(f"{message} for table {table.full_name}: {str(e)}") + logger.info(f"{message} for table {table.full_name}") logger.debug(f"Stack trace for {table.full_name}:", exc_info=e) return [] diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py index 93df5a3d8a382..33596091e420d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py @@ -20,6 +20,7 @@ support_status, ) from datahub.ingestion.api.source import MetadataWorkUnitProcessor, SourceReport +from datahub.ingestion.api.source_helpers import auto_workunit from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.common.subtypes import ( BIContainerSubTypes, @@ -292,8 +293,12 @@ def to_datahub_dataset( """ dataset_mcps: List[MetadataChangeProposalWrapper] = [] + if dataset is None: return dataset_mcps + + logger.debug(f"Processing dataset {dataset.name}") + if not any( [ self.__config.filter_dataset_endorsements.allowed(tag) @@ -310,7 +315,6 @@ def to_datahub_dataset( ) for table in dataset.tables: - self.processed_datasets.add(dataset) # Create a URN for dataset ds_urn = builder.make_dataset_urn_with_platform_instance( platform=self.__config.platform_name, @@ -409,6 +413,8 @@ def to_datahub_dataset( dataset.tags, ) + self.processed_datasets.add(dataset) + return dataset_mcps @staticmethod @@ -1144,6 +1150,21 @@ def validate_dataset_type_mapping(self): f"Dataset lineage would get ingested for data-platform = {self.source_config.dataset_type_mapping}" ) + def extract_datasets_as_containers(self): + for dataset in self.mapper.processed_datasets: + yield from self.mapper.generate_container_for_dataset(dataset) + + def extract_independent_datasets( + self, workspace: powerbi_data_classes.Workspace + ) -> Iterable[MetadataWorkUnit]: + for dataset in workspace.independent_datasets: + yield from auto_workunit( + stream=self.mapper.to_datahub_dataset( + dataset=dataset, + workspace=workspace, + ) + ) + def get_workspace_workunit( self, workspace: powerbi_data_classes.Workspace ) -> Iterable[MetadataWorkUnit]: @@ -1179,11 +1200,10 @@ def get_workspace_workunit( ): yield work_unit - for dataset in self.mapper.processed_datasets: - if self.source_config.extract_datasets_to_containers: - dataset_workunits = self.mapper.generate_container_for_dataset(dataset) - for workunit in dataset_workunits: - yield workunit + if self.source_config.extract_datasets_to_containers: + yield from self.extract_datasets_as_containers() + + yield from self.extract_independent_datasets(workspace) def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: # As modified_workspaces is not idempotent, hence workunit processors are run later for each workspace_id diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py index 91f99c4fff580..28a5fac8b127b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py @@ -46,6 +46,7 @@ class Workspace: report_endorsements: Dict[str, List[str]] dashboard_endorsements: Dict[str, List[str]] scan_result: dict + independent_datasets: List["PowerBIDataset"] def get_urn_part(self, workspace_id_as_urn_part: Optional[bool] = False) -> str: # shouldn't use workspace name, as they can be the same? diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py index 7f8153b108497..17787af38983a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py @@ -206,6 +206,7 @@ def get_workspaces(self) -> List[Workspace]: report_endorsements={}, dashboard_endorsements={}, scan_result={}, + independent_datasets=[], ) for workspace in groups ] @@ -227,6 +228,7 @@ def get_modified_workspaces(self) -> List[Workspace]: report_endorsements={}, dashboard_endorsements={}, scan_result={}, + independent_datasets=[], ) for workspace_id in modified_workspace_ids ] @@ -382,6 +384,7 @@ def _fill_metadata_from_scan_result( report_endorsements={}, dashboard_endorsements={}, scan_result={}, + independent_datasets=[], ) cur_workspace.scan_result = workspace_metadata cur_workspace.datasets = self._get_workspace_datasets( @@ -406,6 +409,29 @@ def _fill_metadata_from_scan_result( return workspaces + def _fill_independent_datasets(self, workspace: Workspace) -> None: + if self.__config.extract_independent_datasets is False: + logger.info( + "Skipping independent datasets retrieval as extract_independent_datasets is set to false" + ) + return + + reachable_datasets: List[str] = [] + # Find out reachable datasets + for dashboard in workspace.dashboards: + for tile in dashboard.tiles: + if tile.dataset is not None: + reachable_datasets.append(tile.dataset.id) + + for report in workspace.reports: + if report.dataset is not None: + reachable_datasets.append(report.dataset.id) + + # Set datasets not present in reachable_datasets + for dataset in workspace.datasets.values(): + if dataset.id not in reachable_datasets: + workspace.independent_datasets.append(dataset) + def _fill_regular_metadata_detail(self, workspace: Workspace) -> None: def fill_dashboards() -> None: workspace.dashboards = self._get_resolver().get_dashboards(workspace) @@ -437,6 +463,7 @@ def fill_dashboard_tags() -> None: fill_reports() fill_dashboard_tags() + self._fill_independent_datasets(workspace=workspace) # flake8: noqa: C901 def fill_workspaces( diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_independent_datasets.json b/metadata-ingestion/tests/integration/powerbi/golden_test_independent_datasets.json new file mode 100644 index 0000000000000..d204d426a38d3 --- /dev/null +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_independent_datasets.json @@ -0,0 +1,73 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,employee-dataset.employee_ctc,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "dummy", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,employee-dataset.employee_ctc,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "91580e0e-1680-4b1c-bbf9-4f6764d7a5ff" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/91580e0e-1680-4b1c-bbf9-4f6764d7a5ff/details", + "name": "employee_ctc", + "description": "Employee Management", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,employee-dataset.employee_ctc,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,employee-dataset.employee_ctc,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Dataset Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py index dbdce2da68278..5036f758a7de9 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py +++ b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py @@ -434,6 +434,19 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: }, ], }, + { + "id": "91580e0e-1680-4b1c-bbf9-4f6764d7a5ff", + "tables": [ + { + "name": "employee_ctc", + "source": [ + { + "expression": "dummy", + } + ], + } + ], + }, ], "dashboards": [ { @@ -566,6 +579,16 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: ] }, }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/91580e0e-1680-4b1c-bbf9-4f6764d7a5ff": { + "method": "GET", + "status_code": 200, + "json": { + "id": "91580e0e-1680-4b1c-bbf9-4f6764d7a5ff", + "name": "employee-dataset", + "description": "Employee Management", + "webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/91580e0e-1680-4b1c-bbf9-4f6764d7a5ff", + }, + }, } api_vs_response.update(override_data) @@ -1209,6 +1232,7 @@ def validate_pipeline(pipeline: Pipeline) -> None: report_endorsements={}, dashboard_endorsements={}, scan_result={}, + independent_datasets=[], ) # Fetch actual reports reports: List[Report] = cast( @@ -1389,3 +1413,95 @@ def test_reports_with_failed_page_request( ) validate_pipeline(pipeline) + + +@freeze_time(FROZEN_TIME) +@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca) +def test_independent_datasets_extraction( + mock_msal, pytestconfig, tmp_path, mock_time, requests_mock +): + + test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" + + register_mock_api( + request_mock=requests_mock, + override_data={ + "https://api.powerbi.com/v1.0/myorg/groups": { + "method": "GET", + "status_code": 200, + "json": { + "@odata.count": 3, + "value": [ + { + "id": "64ED5CAD-7C10-4684-8180-826122881108", + "isReadOnly": True, + "name": "demo-workspace", + "type": "Workspace", + }, + ], + }, + }, + "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/4674efd1-603c-4129-8d82-03cf2be05aff": { + "method": "GET", + "status_code": 200, + "json": { + "workspaces": [ + { + "id": "64ED5CAD-7C10-4684-8180-826122881108", + "name": "demo-workspace", + "state": "Active", + "datasets": [ + { + "id": "91580e0e-1680-4b1c-bbf9-4f6764d7a5ff", + "tables": [ + { + "name": "employee_ctc", + "source": [ + { + "expression": "dummy", + } + ], + } + ], + }, + ], + }, + ] + }, + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards": { + "method": "GET", + "status_code": 200, + "json": {"value": []}, + }, + }, + ) + + pipeline = Pipeline.create( + { + "run_id": "powerbi-test", + "source": { + "type": "powerbi", + "config": { + **default_source_config(), + "extract_independent_datasets": True, + }, + }, + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/powerbi_independent_mces.json", + }, + }, + } + ) + + pipeline.run() + pipeline.raise_from_status() + golden_file = "golden_test_independent_datasets.json" + + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / "powerbi_independent_mces.json", + golden_path=f"{test_resources_dir}/{golden_file}", + ) From 35a44344902cb99cd02910fb785a36acc4860af9 Mon Sep 17 00:00:00 2001 From: Shubham Jagtap <132359390+shubhamjagtap639@users.noreply.github.com> Date: Fri, 16 Jun 2023 02:36:28 +0530 Subject: [PATCH 025/222] fix(sdk/dataflow): deprecate cluster and use env and platform_instance instead (#8201) Co-authored-by: mohdsiddique Co-authored-by: Harshal Sheth --- docs/how/updating-datahub.md | 2 + .../library/lineage_job_dataflow_new_api.py | 2 +- .../lineage_job_dataflow_new_api_simple.py | 2 +- .../lineage_job_dataflow_new_api_verbose.py | 2 +- .../datahub/api/entities/datajob/dataflow.py | 56 ++++++++++++++++++- .../dataprocess/dataprocess_instance.py | 6 +- .../client/airflow_generator.py | 2 +- 7 files changed, 62 insertions(+), 10 deletions(-) diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 4f8e193466778..59cc1348cb4cb 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -6,6 +6,8 @@ This file documents any backwards-incompatible changes in DataHub and assists pe ### Breaking Changes +- #8201: Python SDK: In the DataFlow class, the `cluster` argument is deprecated in favor of `env`. + ### Potential Downtime ### Deprecations diff --git a/metadata-ingestion/examples/library/lineage_job_dataflow_new_api.py b/metadata-ingestion/examples/library/lineage_job_dataflow_new_api.py index 60f2a11d22fdd..7318544d1beb4 100644 --- a/metadata-ingestion/examples/library/lineage_job_dataflow_new_api.py +++ b/metadata-ingestion/examples/library/lineage_job_dataflow_new_api.py @@ -5,7 +5,7 @@ emitter = DatahubRestEmitter("http://localhost:8080") -jobFlow = DataFlow(cluster="prod", orchestrator="airflow", id="flow_new_api") +jobFlow = DataFlow(env="prod", orchestrator="airflow", id="flow_new_api") jobFlow.emit(emitter) dataJob = DataJob(flow_urn=jobFlow.urn, id="job1") diff --git a/metadata-ingestion/examples/library/lineage_job_dataflow_new_api_simple.py b/metadata-ingestion/examples/library/lineage_job_dataflow_new_api_simple.py index 1871a8af09e50..d47cdbce14303 100644 --- a/metadata-ingestion/examples/library/lineage_job_dataflow_new_api_simple.py +++ b/metadata-ingestion/examples/library/lineage_job_dataflow_new_api_simple.py @@ -10,7 +10,7 @@ emitter = DatahubRestEmitter("http://localhost:8080") -jobFlow = DataFlow(cluster="prod", orchestrator="airflow", id="flow_api_simple") +jobFlow = DataFlow(env="prod", orchestrator="airflow", id="flow_api_simple") jobFlow.emit(emitter) # Flowurn as constructor diff --git a/metadata-ingestion/examples/library/lineage_job_dataflow_new_api_verbose.py b/metadata-ingestion/examples/library/lineage_job_dataflow_new_api_verbose.py index 97acdbee1c99c..67a8ce2059679 100644 --- a/metadata-ingestion/examples/library/lineage_job_dataflow_new_api_verbose.py +++ b/metadata-ingestion/examples/library/lineage_job_dataflow_new_api_verbose.py @@ -13,7 +13,7 @@ emitter = DatahubRestEmitter("http://localhost:8080") -jobFlow = DataFlow(cluster="prod", orchestrator="airflow", id="flow2") +jobFlow = DataFlow(env="prod", orchestrator="airflow", id="flow2") jobFlow.emit(emitter) # Flowurn as constructor diff --git a/metadata-ingestion/src/datahub/api/entities/datajob/dataflow.py b/metadata-ingestion/src/datahub/api/entities/datajob/dataflow.py index 0dd16a6e21f94..fdb9a30083ca4 100644 --- a/metadata-ingestion/src/datahub/api/entities/datajob/dataflow.py +++ b/metadata-ingestion/src/datahub/api/entities/datajob/dataflow.py @@ -1,5 +1,16 @@ +import logging from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Set, Union +from typing import ( + TYPE_CHECKING, + Callable, + Dict, + Iterable, + List, + Optional, + Set, + Union, + cast, +) import datahub.emitter.mce_builder as builder from datahub.emitter.mcp import MetadataChangeProposalWrapper @@ -22,23 +33,62 @@ from datahub.emitter.kafka_emitter import DatahubKafkaEmitter from datahub.emitter.rest_emitter import DatahubRestEmitter +logger = logging.getLogger(__name__) + @dataclass class DataFlow: + """The DataHub representation of data-flow. + + Args: + urn (int): Unique identifier of the DataFlow in DataHub. For more detail refer https://datahubproject.io/docs/what/urn/. + id (str): Identifier of DataFlow in orchestrator. + orchestrator (str): orchestrator. for example airflow. + cluster (Optional[str]): [deprecated] Please use env. + name (str): Name of the DataFlow. + description (str): Description about DataFlow + properties (Optional[str]): Additional properties if any. + url (Optional[str]): URL pointing to DataFlow. + tags (Set[str]): tags that need to be apply on DataFlow. + owners (Set[str]): owners that need to be apply on DataFlow. + platform_instance (Optional[str]): The instance of the platform that all assets produced by this orchestrator belong to. For more detail refer https://datahubproject.io/docs/platform-instances/. + env (Optional[str]): The environment that all assets produced by this orchestrator belong to. For more detail and possible values refer https://datahubproject.io/docs/graphql/enums/#fabrictype. + """ + urn: DataFlowUrn = field(init=False) id: str orchestrator: str - cluster: str + cluster: Optional[str] = None # Deprecated in favor of env name: Optional[str] = None description: Optional[str] = None properties: Dict[str, str] = field(default_factory=dict) url: Optional[str] = None tags: Set[str] = field(default_factory=set) owners: Set[str] = field(default_factory=set) + platform_instance: Optional[str] = None + env: Optional[str] = None def __post_init__(self): + if self.env is not None and self.cluster is not None: + raise ValueError( + "Cannot provide both env and cluster parameter. Cluster is deprecated in favor of env." + ) + + if self.env is None and self.cluster is None: + raise ValueError("env is required") + + if self.cluster is not None: + logger.warning( + "The cluster argument is deprecated. Use env and possibly platform_instance instead." + ) + self.env = self.cluster + + # self.env is optional, cast is required to pass lint self.urn = DataFlowUrn.create_from_ids( - orchestrator=self.orchestrator, env=self.cluster, flow_id=self.id + orchestrator=self.orchestrator, + env=cast(str, self.env), + flow_id=self.id, + platform_instance=self.platform_instance, ) def generate_ownership_aspect(self): diff --git a/metadata-ingestion/src/datahub/api/entities/dataprocess/dataprocess_instance.py b/metadata-ingestion/src/datahub/api/entities/dataprocess/dataprocess_instance.py index fb276684d3b99..9ec389c3a0989 100644 --- a/metadata-ingestion/src/datahub/api/entities/dataprocess/dataprocess_instance.py +++ b/metadata-ingestion/src/datahub/api/entities/dataprocess/dataprocess_instance.py @@ -1,7 +1,7 @@ import time from dataclasses import dataclass, field from enum import Enum -from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Union +from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Union, cast from datahub.api.entities.datajob import DataFlow, DataJob from datahub.emitter.mcp import MetadataChangeProposalWrapper @@ -128,7 +128,7 @@ def emit_process_start( if isinstance(self.template_urn, DataFlowUrn): job_flow_urn = self.template_urn template_object = DataFlow( - cluster=self.template_urn.get_env(), + env=self.template_urn.get_env(), orchestrator=self.template_urn.get_orchestrator_name(), id=self.template_urn.get_flow_id(), ) @@ -326,7 +326,7 @@ def from_dataflow(dataflow: DataFlow, id: str) -> "DataProcessInstance": dpi = DataProcessInstance( id=id, orchestrator=dataflow.orchestrator, - cluster=dataflow.cluster, + cluster=cast(str, dataflow.env), template_urn=dataflow.urn, ) dpi._template_object = dataflow diff --git a/metadata-ingestion/src/datahub_provider/client/airflow_generator.py b/metadata-ingestion/src/datahub_provider/client/airflow_generator.py index 46fff6093cde6..d2d29b00d244f 100644 --- a/metadata-ingestion/src/datahub_provider/client/airflow_generator.py +++ b/metadata-ingestion/src/datahub_provider/client/airflow_generator.py @@ -146,7 +146,7 @@ def generate_dataflow( orchestrator = "airflow" description = f"{dag.description}\n\n{dag.doc_md or ''}" data_flow = DataFlow( - cluster=cluster, id=id, orchestrator=orchestrator, description=description + env=cluster, id=id, orchestrator=orchestrator, description=description ) flow_property_bag: Dict[str, str] = {} From 2751a09284143595bda73e625558de24c29ce26a Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Thu, 15 Jun 2023 23:10:15 -0400 Subject: [PATCH 026/222] fix(ingest): pass platform correctly to browse path v2 helper (#8244) --- .../src/datahub/ingestion/api/source.py | 17 +++++------- .../datahub/ingestion/api/source_helpers.py | 26 +++++++++---------- .../tests/unit/test_source_helpers.py | 5 ++-- 3 files changed, 21 insertions(+), 27 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/api/source.py b/metadata-ingestion/src/datahub/ingestion/api/source.py index 4a90d4b05fbc2..fa037f8d7328b 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/source.py +++ b/metadata-ingestion/src/datahub/ingestion/api/source.py @@ -23,7 +23,7 @@ from datahub.configuration.common import ConfigModel from datahub.configuration.source_common import PlatformInstanceConfigMixin -from datahub.emitter.mcp_builder import PlatformKey, mcps_from_mce +from datahub.emitter.mcp_builder import mcps_from_mce from datahub.ingestion.api.closeable import Closeable from datahub.ingestion.api.common import PipelineContext, RecordEnvelope, WorkUnit from datahub.ingestion.api.report import Report @@ -249,19 +249,14 @@ def _get_browse_path_processor(self, dry_run: bool) -> MetadataWorkUnitProcessor env and env.lower(), ] - platform_key: Optional[PlatformKey] = None - if ( - platform - and isinstance(config, PlatformInstanceConfigMixin) - and config.platform_instance - ): - platform_key = PlatformKey( - platform=platform, instance=config.platform_instance - ) + platform_instance: Optional[str] = None + if isinstance(config, PlatformInstanceConfigMixin) and config.platform_instance: + platform_instance = platform_instance return partial( auto_browse_path_v2, - platform_key=platform_key, + platform=platform, + platform_instance=platform_instance, drop_dirs=[s for s in browse_path_drop_dirs if s is not None], dry_run=dry_run, ) diff --git a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py index 2e6e002a55cbe..32aa6fc7e3b4e 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py +++ b/metadata-ingestion/src/datahub/ingestion/api/source_helpers.py @@ -15,7 +15,6 @@ from datahub.emitter.mce_builder import make_dataplatform_instance_urn from datahub.emitter.mcp import MetadataChangeProposalWrapper -from datahub.emitter.mcp_builder import PlatformKey from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.metadata.schema_classes import ( BrowsePathEntryClass, @@ -173,7 +172,8 @@ def auto_browse_path_v2( *, dry_run: bool = False, drop_dirs: Sequence[str] = (), - platform_key: Optional[PlatformKey] = None, + platform: Optional[str] = None, + platform_instance: Optional[str] = None, ) -> Iterable[MetadataWorkUnit]: """Generate BrowsePathsV2 from Container and BrowsePaths aspects. @@ -248,7 +248,9 @@ def auto_browse_path_v2( yield MetadataChangeProposalWrapper( entityUrn=urn, aspect=BrowsePathsV2Class( - path=_prepend_platform_instance(path, platform_key) + path=_prepend_platform_instance( + path, platform, platform_instance + ) ), ).as_workunit() elif urn not in emitted_urns and guess_entity_type(urn) == "container": @@ -258,16 +260,14 @@ def auto_browse_path_v2( yield MetadataChangeProposalWrapper( entityUrn=urn, aspect=BrowsePathsV2Class( - path=_prepend_platform_instance([], platform_key) + path=_prepend_platform_instance([], platform, platform_instance) ), ).as_workunit() if num_out_of_batch or num_out_of_order: properties = { - "platform": platform_key.platform if platform_key else None, - "has_platform_instance": bool(platform_key.instance) - if platform_key - else False, + "platform": platform, + "has_platform_instance": bool(platform_instance), "num_out_of_batch": num_out_of_batch, "num_out_of_order": num_out_of_order, } @@ -293,12 +293,12 @@ def _batch_workunits_by_urn( def _prepend_platform_instance( - entries: List[BrowsePathEntryClass], platform_key: Optional[PlatformKey] + entries: List[BrowsePathEntryClass], + platform: Optional[str], + platform_instance: Optional[str], ) -> List[BrowsePathEntryClass]: - if platform_key and platform_key.instance: - urn = make_dataplatform_instance_urn( - platform_key.platform, platform_key.instance - ) + if platform and platform_instance: + urn = make_dataplatform_instance_urn(platform, platform_instance) return [BrowsePathEntryClass(id=urn, urn=urn)] + entries return entries diff --git a/metadata-ingestion/tests/unit/test_source_helpers.py b/metadata-ingestion/tests/unit/test_source_helpers.py index 9f88f0d635958..dbf0c3ea50a50 100644 --- a/metadata-ingestion/tests/unit/test_source_helpers.py +++ b/metadata-ingestion/tests/unit/test_source_helpers.py @@ -8,7 +8,6 @@ make_dataset_urn, ) from datahub.emitter.mcp import MetadataChangeProposalWrapper -from datahub.emitter.mcp_builder import PlatformKey from datahub.ingestion.api.source_helpers import ( auto_browse_path_v2, auto_status_aspect, @@ -298,7 +297,6 @@ def test_auto_browse_path_v2_container_over_legacy_browse_path(telemetry_ping_mo def test_auto_browse_path_v2_with_platform_instsance(telemetry_ping_mock): platform = "my_platform" platform_instance = "my_instance" - platform_key = PlatformKey(platform=platform, instance=platform_instance) platform_instance_urn = make_dataplatform_instance_urn(platform, platform_instance) platform_instance_entry = models.BrowsePathEntryClass( platform_instance_urn, platform_instance_urn @@ -310,7 +308,8 @@ def test_auto_browse_path_v2_with_platform_instsance(telemetry_ping_mock): new_wus = list( auto_browse_path_v2( wus, - platform_key=platform_key, + platform=platform, + platform_instance=platform_instance, ) ) assert telemetry_ping_mock.call_count == 0 From d1ff9024b90829e00e1b1cb9666a7f7a683f6f10 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Fri, 16 Jun 2023 01:03:29 -0700 Subject: [PATCH 027/222] feat(search): Supporting Aggregations for hasX fields (#8241) --- .../models/SearchableFieldSpecExtractor.java | 15 +++- .../annotation/SearchableAnnotation.java | 38 +++++++-- .../request/AggregationQueryBuilder.java | 84 +++++++++++++------ .../query/request/SearchRequestHandler.java | 32 ++++--- .../request/AggregationQueryBuilderTest.java | 73 ++++++++++++++++ 5 files changed, 196 insertions(+), 46 deletions(-) create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilderTest.java diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/SearchableFieldSpecExtractor.java b/entity-registry/src/main/java/com/linkedin/metadata/models/SearchableFieldSpecExtractor.java index 5cd7be4d68d73..2ffd9283ed456 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/SearchableFieldSpecExtractor.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/SearchableFieldSpecExtractor.java @@ -143,9 +143,18 @@ private void extractSearchableAnnotation(final Object annotationObj, final DataS throw new ModelValidationException( String.format("Entity has multiple searchable fields with the same field name %s, path: %s", annotation.getFieldName(), fullPath.orElse(path))); } else { - annotation = new SearchableAnnotation(pathName, annotation.getFieldType(), annotation.isQueryByDefault(), - annotation.isEnableAutocomplete(), annotation.isAddToFilters(), annotation.getFilterNameOverride(), - annotation.getBoostScore(), annotation.getHasValuesFieldName(), annotation.getNumValuesFieldName(), + annotation = new SearchableAnnotation( + pathName, + annotation.getFieldType(), + annotation.isQueryByDefault(), + annotation.isEnableAutocomplete(), + annotation.isAddToFilters(), + annotation.isAddHasValuesToFilters(), + annotation.getFilterNameOverride(), + annotation.getHasValuesFilterNameOverride(), + annotation.getBoostScore(), + annotation.getHasValuesFieldName(), + annotation.getNumValuesFieldName(), annotation.getWeightsPerFieldValue()); } } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/annotation/SearchableAnnotation.java b/entity-registry/src/main/java/com/linkedin/metadata/models/annotation/SearchableAnnotation.java index 9ac77781544b3..f2e65c771c6eb 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/annotation/SearchableAnnotation.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/annotation/SearchableAnnotation.java @@ -33,8 +33,12 @@ public class SearchableAnnotation { boolean enableAutocomplete; // Whether or not to add field to filters. boolean addToFilters; - // Name of the filter + // Whether or not to add the "has values" to filters. + boolean addHasValuesToFilters; + // Display name of the filter Optional filterNameOverride; + // Display name of the has values filter + Optional hasValuesFilterNameOverride; // Boost multiplier to the match score. Matches on fields with higher boost score ranks higher double boostScore; // If set, add a index field of the given name that checks whether the field exists @@ -80,18 +84,29 @@ public static SearchableAnnotation fromPegasusAnnotationObject(@Nonnull final Ob final Optional queryByDefault = AnnotationUtils.getField(map, "queryByDefault", Boolean.class); final Optional enableAutocomplete = AnnotationUtils.getField(map, "enableAutocomplete", Boolean.class); final Optional addToFilters = AnnotationUtils.getField(map, "addToFilters", Boolean.class); + final Optional addHasValuesToFilters = AnnotationUtils.getField(map, "addHasValuesToFilters", Boolean.class); final Optional filterNameOverride = AnnotationUtils.getField(map, "filterNameOverride", String.class); + final Optional hasValuesFilterNameOverride = + AnnotationUtils.getField(map, "hasValuesFilterNameOverride", String.class); final Optional boostScore = AnnotationUtils.getField(map, "boostScore", Double.class); final Optional hasValuesFieldName = AnnotationUtils.getField(map, "hasValuesFieldName", String.class); final Optional numValuesFieldName = AnnotationUtils.getField(map, "numValuesFieldName", String.class); - final Optional weightsPerFieldValueMap = AnnotationUtils.getField(map, "weightsPerFieldValue", Map.class).map(m -> (Map) m); final FieldType resolvedFieldType = getFieldType(fieldType, schemaDataType); - return new SearchableAnnotation(fieldName.orElse(schemaFieldName), resolvedFieldType, - getQueryByDefault(queryByDefault, resolvedFieldType), enableAutocomplete.orElse(false), - addToFilters.orElse(false), filterNameOverride, boostScore.orElse(1.0), hasValuesFieldName, numValuesFieldName, + return new SearchableAnnotation( + fieldName.orElse(schemaFieldName), + resolvedFieldType, + getQueryByDefault(queryByDefault, resolvedFieldType), + enableAutocomplete.orElse(false), + addToFilters.orElse(false), + addHasValuesToFilters.orElse(false), + filterNameOverride, + hasValuesFilterNameOverride, + boostScore.orElse(1.0), + hasValuesFieldName, + numValuesFieldName, weightsPerFieldValueMap.orElse(ImmutableMap.of())); } @@ -127,4 +142,17 @@ private static Boolean getQueryByDefault(Optional maybeQueryByDefault, public String getFilterName() { return filterNameOverride.orElse(fieldName); } + + public String getHasValuesFilterName() { + return hasValuesFilterNameOverride.orElse( + hasValuesFieldName.orElse(String.format("has%s", capitalizeFirstLetter(fieldName)))); + } + + private static String capitalizeFirstLetter(String str) { + if (str == null || str.length() == 0) { + return str; + } else { + return str.substring(0, 1).toUpperCase() + str.substring(1); + } + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java index 213179faae56f..2e080747a899d 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java @@ -3,9 +3,12 @@ import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.models.annotation.SearchableAnnotation; import com.linkedin.metadata.search.utils.ESUtils; +import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; import org.elasticsearch.search.aggregations.AggregationBuilder; @@ -16,34 +19,55 @@ @Slf4j public class AggregationQueryBuilder { + private final SearchConfiguration _configs; - private final List _annotations; private final Set _facetFields; - public AggregationQueryBuilder(SearchConfiguration configs, List annotations) { - this._configs = configs; - this._annotations = annotations; + + public AggregationQueryBuilder( + @Nonnull final SearchConfiguration configs, + @Nonnull final List annotations) { + this._configs = Objects.requireNonNull(configs, "configs must not be null"); this._facetFields = getFacetFields(annotations); } - private Set getFacetFields(List annotations) { + /** + * Get the set of default aggregations, across all facets. + */ + public List getAggregations() { + return getAggregations(null); + } + + /** + * Get aggregations for a search request for the given facets provided, and if none are provided, then get aggregations for all. + */ + public List getAggregations(@Nullable List facets) { + final Set facetsToAggregate; + if (facets != null) { + facets.stream().filter(f -> !isValidAggregate(f)).forEach(facet -> { + log.warn(String.format("Provided facet for search filter aggregations that doesn't exist. Provided: %s; Available: %s", facet, _facetFields)); + }); + facetsToAggregate = facets.stream().filter(this::isValidAggregate).collect(Collectors.toSet()); + } else { + facetsToAggregate = _facetFields; + } + return facetsToAggregate.stream().map(this::facetToAggregationBuilder).collect(Collectors.toList()); + } + + + private Set getFacetFields(final List annotations) { Set facets = annotations.stream() - .filter(SearchableAnnotation::isAddToFilters) - .map(SearchableAnnotation::getFieldName) + .flatMap(annotation -> getFacetFieldsFromAnnotation(annotation).stream()) .collect(Collectors.toSet()); facets.add(INDEX_VIRTUAL_FIELD); return facets; } - public List getAggregations() { - return getAggregations(null); - } - - private boolean isValidAggregate(String inputFacet) { + private boolean isValidAggregate(final String inputFacet) { Set facets = Set.of(inputFacet.split(AGGREGATION_SEPARATOR_CHAR)); return facets.size() > 0 && _facetFields.containsAll(facets); } - private AggregationBuilder facetToAggregationBuilder(String inputFacet) { + private AggregationBuilder facetToAggregationBuilder(final String inputFacet) { List facets = List.of(inputFacet.split(AGGREGATION_SEPARATOR_CHAR)); AggregationBuilder lastAggBuilder = null; for (int i = facets.size() - 1; i >= 0; i--) { @@ -52,7 +76,9 @@ private AggregationBuilder facetToAggregationBuilder(String inputFacet) { facet = "_index"; } AggregationBuilder aggBuilder = - AggregationBuilders.terms(inputFacet).field(ESUtils.toKeywordField(facet, false)).size(_configs.getMaxTermBucketSize()); + AggregationBuilders.terms(inputFacet) + .field(getAggregationField(facet)) + .size(_configs.getMaxTermBucketSize()); if (lastAggBuilder != null) { aggBuilder = aggBuilder.subAggregation(lastAggBuilder); } @@ -62,19 +88,23 @@ private AggregationBuilder facetToAggregationBuilder(String inputFacet) { return lastAggBuilder; } - /** - * Get aggregations for a search request for the given facets provided, and if none are provided, then get aggregations for all. - */ - public List getAggregations(@Nullable List facets) { - final Set facetsToAggregate; - if (facets != null) { - facets.stream().filter(f -> !isValidAggregate(f)).forEach(facet -> { - log.warn(String.format("Provided facet for search filter aggregations that doesn't exist. Provided: %s; Available: %s", facet, _facetFields)); - }); - facetsToAggregate = facets.stream().filter(this::isValidAggregate).collect(Collectors.toSet()); - } else { - facetsToAggregate = _facetFields; + private String getAggregationField(final String facet) { + if (facet.startsWith("has")) { + // Boolean hasX field, not a keyword field. Return the name of the original facet. + return facet; } - return facetsToAggregate.stream().map(this::facetToAggregationBuilder).collect(Collectors.toList()); + // Otherwise assume that this field is of keyword type. + return ESUtils.toKeywordField(facet, false); + } + + List getFacetFieldsFromAnnotation(final SearchableAnnotation annotation) { + final List facetsFromAnnotation = new ArrayList<>(); + if (annotation.isAddToFilters()) { + facetsFromAnnotation.add(annotation.getFieldName()); + } + if (annotation.isAddHasValuesToFilters() && annotation.getHasValuesFieldName().isPresent()) { + facetsFromAnnotation.add(annotation.getHasValuesFieldName().get()); + } + return facetsFromAnnotation; } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java index 71cc97c58c4f3..7048f7ed955bf 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java @@ -31,6 +31,7 @@ import com.linkedin.metadata.search.features.Features; import com.linkedin.metadata.search.utils.ESUtils; import com.linkedin.metadata.utils.SearchUtil; +import com.linkedin.util.Pair; import io.opentelemetry.extension.annotations.WithSpan; import java.net.URISyntaxException; import java.util.ArrayList; @@ -81,11 +82,11 @@ public class SearchRequestHandler { .setSkipCache(false) .setSkipAggregates(false) .setSkipHighlighting(false); - private static final Map, SearchRequestHandler> REQUEST_HANDLER_BY_ENTITY_NAME = new ConcurrentHashMap<>(); private static final String REMOVED = "removed"; - private static final String URN_FILTER = "urn"; + private static final String[] FIELDS_TO_FETCH = new String[]{"urn", "usageCountLast30Days"}; + private static final String[] URN_FIELD = new String[]{"urn"}; private final List _entitySpecs; private final Set _defaultQueryFieldNames; @@ -106,8 +107,8 @@ private SearchRequestHandler(@Nonnull List entitySpecs, @Nonnull Sea List annotations = getSearchableAnnotations(); _defaultQueryFieldNames = getDefaultQueryFieldNames(annotations); _filtersToDisplayName = annotations.stream() - .filter(SearchableAnnotation::isAddToFilters) - .collect(Collectors.toMap(SearchableAnnotation::getFieldName, SearchableAnnotation::getFilterName, mapMerger())); + .flatMap(annotation -> getFacetFieldDisplayNameFromAnnotation(annotation).stream()) + .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond, mapMerger())); _filtersToDisplayName.put(INDEX_VIRTUAL_FIELD, "Type"); _highlights = getHighlights(); _searchQueryBuilder = new SearchQueryBuilder(configs, customSearchConfiguration); @@ -135,13 +136,6 @@ private List getSearchableAnnotations() { .collect(Collectors.toList()); } - private Set getFacetFields(List annotations) { - return annotations.stream() - .filter(SearchableAnnotation::isAddToFilters) - .map(SearchableAnnotation::getFieldName) - .collect(Collectors.toSet()); - } - @VisibleForTesting private Set getDefaultQueryFieldNames(List annotations) { return Stream.concat(annotations.stream() @@ -695,4 +689,20 @@ private AggregationMetadata buildAggregationMetadata( .setFilterValues(filterValues); } + private List> getFacetFieldDisplayNameFromAnnotation( + @Nonnull final SearchableAnnotation annotation + ) { + final List> facetsFromAnnotation = new ArrayList<>(); + // Case 1: Default Keyword field + if (annotation.isAddToFilters()) { + facetsFromAnnotation.add(Pair.of(annotation.getFieldName(), annotation.getFilterName())); + } + // Case 2: HasX boolean field + if (annotation.isAddHasValuesToFilters() && annotation.getHasValuesFieldName().isPresent()) { + facetsFromAnnotation.add(Pair.of( + annotation.getHasValuesFieldName().get(), annotation.getHasValuesFilterName() + )); + } + return facetsFromAnnotation; + } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilderTest.java new file mode 100644 index 0000000000000..914911677c2c9 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilderTest.java @@ -0,0 +1,73 @@ +package com.linkedin.metadata.search.elasticsearch.query.request; + +import com.google.common.collect.ImmutableList; +import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.models.annotation.SearchableAnnotation; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.testng.Assert; +import org.testng.annotations.Test; + + +public class AggregationQueryBuilderTest { + + @Test + public void testGetAggregationsHasFields() { + + SearchableAnnotation annotation = new SearchableAnnotation( + "test", + SearchableAnnotation.FieldType.KEYWORD, + true, + true, + false, + true, + Optional.empty(), + Optional.of("Has Test"), + 1.0, + Optional.of("hasTest"), + Optional.empty(), + Collections.emptyMap() + ); + + SearchConfiguration config = new SearchConfiguration(); + config.setMaxTermBucketSize(25); + + AggregationQueryBuilder builder = new AggregationQueryBuilder( + config, ImmutableList.of(annotation)); + + List aggs = builder.getAggregations(); + + Assert.assertTrue(aggs.stream().anyMatch(agg -> agg.getName().equals("hasTest"))); + } + + @Test + public void testGetAggregationsFields() { + + SearchableAnnotation annotation = new SearchableAnnotation( + "test", + SearchableAnnotation.FieldType.KEYWORD, + true, + true, + true, + false, + Optional.of("Test Filter"), + Optional.empty(), + 1.0, + Optional.empty(), + Optional.empty(), + Collections.emptyMap() + ); + + SearchConfiguration config = new SearchConfiguration(); + config.setMaxTermBucketSize(25); + + AggregationQueryBuilder builder = new AggregationQueryBuilder( + config, ImmutableList.of(annotation)); + + List aggs = builder.getAggregations(); + + Assert.assertTrue(aggs.stream().anyMatch(agg -> agg.getName().equals("test"))); + } +} From e35ac44f58cca1ae3db833ffbc5af7cf9d358407 Mon Sep 17 00:00:00 2001 From: Indy Prentice Date: Fri, 16 Jun 2023 11:47:19 -0500 Subject: [PATCH 028/222] fix(ingest): Call validator on the base urn as well as aspect components when ingesting (#8250) Co-authored-by: Indy Prentice --- .../com/linkedin/metadata/entity/EntityService.java | 10 +++++++--- .../linkedin/metadata/entity/EntityServiceTest.java | 7 ++----- .../resources/index/usage-event/aws_es_ism_policy.json | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java index fca9960d94994..97e7952f65a77 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java @@ -10,6 +10,7 @@ import com.github.fge.jsonpatch.JsonPatch; import com.github.fge.jsonpatch.JsonPatchException; import com.github.fge.jsonpatch.Patch; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterators; @@ -24,6 +25,7 @@ import com.linkedin.common.urn.VersionedUrnUtils; import com.linkedin.data.schema.RecordDataSchema; import com.linkedin.data.schema.TyperefDataSchema; +import com.linkedin.data.schema.validation.ValidationResult; import com.linkedin.data.schema.validator.Validator; import com.linkedin.data.template.DataTemplateUtil; import com.linkedin.data.template.RecordTemplate; @@ -563,10 +565,11 @@ private void validateAspect(Urn urn, RecordTemplate aspect) { } private void validateAspect(Urn urn, RecordTemplate aspect, Validator validator) { - RecordTemplateValidator.validate(aspect, validationResult -> { + Consumer resultFunction = validationResult -> { throw new IllegalArgumentException("Invalid format for aspect: " + aspect + " for entity: " + urn + "\n Cause: " - + validationResult.getMessages()); - }, validator); + + validationResult.getMessages()); }; + RecordTemplateValidator.validate(buildKeyAspect(urn), resultFunction, validator); + RecordTemplateValidator.validate(aspect, resultFunction, validator); } /** * Checks whether there is an actual update to the aspect by applying the updateLambda @@ -697,6 +700,7 @@ protected SystemMetadata generateSystemMetadataIfEmpty(@Nullable SystemMetadata return systemMetadata; } + @VisibleForTesting static void validateUrn(@Nonnull final Urn urn) { if (urn.toString().trim().length() != urn.toString().length()) { diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java index 117ed92e6f468..dee6ccb2f9b2e 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java @@ -1215,12 +1215,9 @@ public void testValidateUrn() throws Exception { } // Urn purely too long - StringBuilder buildStringTooLong = new StringBuilder(); - for (int i = 0; i < 510; i++) { - buildStringTooLong.append('a'); - } + String stringTooLong = "a".repeat(510); - Urn testUrnTooLong = new Urn("li", "testType", new TupleKey(buildStringTooLong.toString())); + Urn testUrnTooLong = new Urn("li", "testType", new TupleKey(stringTooLong)); try { EntityService.validateUrn(testUrnTooLong); Assert.fail("Should have raised IllegalArgumentException for URN too long"); diff --git a/metadata-service/restli-servlet-impl/src/main/resources/index/usage-event/aws_es_ism_policy.json b/metadata-service/restli-servlet-impl/src/main/resources/index/usage-event/aws_es_ism_policy.json index c1ab584a1ce61..bc1442d216918 100644 --- a/metadata-service/restli-servlet-impl/src/main/resources/index/usage-event/aws_es_ism_policy.json +++ b/metadata-service/restli-servlet-impl/src/main/resources/index/usage-event/aws_es_ism_policy.json @@ -56,4 +56,4 @@ "priority": 100 } } -} \ No newline at end of file +} From ec37cfa3bfe83d8536eade96c08b819c7c3d0077 Mon Sep 17 00:00:00 2001 From: Jeff Merrick Date: Fri, 16 Jun 2023 12:31:43 -0500 Subject: [PATCH 029/222] docs(website): adjust markprompt z-index so it's not covered by nav (#8255) --- .../src/components/MarkpromptHelp/markprompthelp.module.scss | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs-website/src/components/MarkpromptHelp/markprompthelp.module.scss b/docs-website/src/components/MarkpromptHelp/markprompthelp.module.scss index ef15520a92c28..270877cd04a9f 100644 --- a/docs-website/src/components/MarkpromptHelp/markprompthelp.module.scss +++ b/docs-website/src/components/MarkpromptHelp/markprompthelp.module.scss @@ -73,12 +73,14 @@ button { .MarkpromptOverlay { position: fixed; + z-index: calc(var(--ifm-z-index-fixed) + 1); inset: 0; animation: overlayShow 150ms cubic-bezier(0.16, 1, 0.3, 1); background-color: var(--markprompt-overlay); } .MarkpromptContent { + z-index: calc(var(--ifm-z-index-fixed) + 2); background-color: var(--markprompt-muted); border-radius: var(--markprompt-radius); border: 1px solid var(--markprompt-border); @@ -89,7 +91,7 @@ button { transform: translate(-50%, -50%); width: 80vw; max-width: 600px; - height: calc(100vh - 200px); + height: calc(100vh - 100px); max-height: 600px; animation-name: contentShow; animation-duration: 300ms; From 18b5a916b3c9122bcbe5876141e43fa837e9d628 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Fri, 16 Jun 2023 11:16:59 -0700 Subject: [PATCH 030/222] fix(patch): Fix exception when using default patch for patching missing aspects (#8221) --- .../metadata/entity/EntityService.java | 33 ++++++++----------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java index 97e7952f65a77..12f35532aa65f 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java @@ -622,28 +622,21 @@ protected UpdateAspectResult patchAspectToLocalDB( return _aspectDao.runInTransactionWithRetry(() -> { final String urnStr = urn.toString(); final String aspectName = aspectSpec.getName(); - EntityAspect latest = _aspectDao.getLatestAspect(urnStr, aspectName); - if (latest == null) { - //TODO: best effort mint - RecordTemplate defaultTemplate = _entityRegistry.getAspectTemplateEngine().getDefaultTemplate(aspectSpec.getName()); - - if (defaultTemplate != null) { - latest = new EntityAspect(); - latest.setAspect(aspectName); - latest.setMetadata(EntityUtils.toJsonAspect(defaultTemplate)); - latest.setUrn(urnStr); - latest.setVersion(ASPECT_LATEST_VERSION); - latest.setCreatedOn(new Timestamp(auditStamp.getTime())); - latest.setCreatedBy(auditStamp.getActor().toString()); - } else { - throw new UnsupportedOperationException("Patch not supported for empty aspect for aspect name: " + aspectName); + final EntityAspect latest = _aspectDao.getLatestAspect(urnStr, aspectName); + final long nextVersion = _aspectDao.getNextVersion(urnStr, aspectName); + try { + + final RecordTemplate currentValue = latest != null + ? EntityUtils.toAspectRecord(urn, aspectName, latest.getMetadata(), _entityRegistry) + : _entityRegistry.getAspectTemplateEngine().getDefaultTemplate(aspectSpec.getName()); + + if (latest == null && currentValue == null) { + // Attempting to patch a value to an aspect which has no default value and no existing value. + throw new UnsupportedOperationException(String.format("Patch not supported for aspect with name %s. " + + "Default aspect is required because no aspect currently exists for urn %s.", aspectName, urn)); } - } - long nextVersion = _aspectDao.getNextVersion(urnStr, aspectName); - try { - RecordTemplate currentValue = EntityUtils.toAspectRecord(urn, aspectName, latest.getMetadata(), _entityRegistry); - RecordTemplate updatedValue = _entityRegistry.getAspectTemplateEngine().applyPatch(currentValue, jsonPatch, aspectSpec); + final RecordTemplate updatedValue = _entityRegistry.getAspectTemplateEngine().applyPatch(currentValue, jsonPatch, aspectSpec); validateAspect(urn, updatedValue); return ingestAspectToLocalDBNoTransaction(urn, aspectName, ignored -> updatedValue, auditStamp, providedSystemMetadata, From a5ee1cae6229260b2d02bd671d2148941bfd8567 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Fri, 16 Jun 2023 17:26:30 -0500 Subject: [PATCH 031/222] fix(custom-search): revert underscore as quoted (#8163) --- .../factories/src/main/resources/search_config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata-service/factories/src/main/resources/search_config.yml b/metadata-service/factories/src/main/resources/search_config.yml index 8d65c42dd525e..331e7d8571476 100644 --- a/metadata-service/factories/src/main/resources/search_config.yml +++ b/metadata-service/factories/src/main/resources/search_config.yml @@ -29,9 +29,9 @@ queryConfigurations: boost_mode: multiply # Criteria for exact-match only - # Contains quoted or contains underscore then use exact match query + # Contains quoted then use exact match query - queryRegex: >- - ["'].+["']|\S+_\S+ + ["'].+["'] simpleQuery: false prefixMatchQuery: true exactMatchQuery: true From 3de94c52230ff2ea25de6afd1cf42c7fd85b2375 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Mon, 19 Jun 2023 13:47:44 +0530 Subject: [PATCH 032/222] chore(ci): add back optional static sleep for tests (#8258) --- smoke-test/tests/consistency_utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/smoke-test/tests/consistency_utils.py b/smoke-test/tests/consistency_utils.py index 1d1f3ac58297f..15993733c592b 100644 --- a/smoke-test/tests/consistency_utils.py +++ b/smoke-test/tests/consistency_utils.py @@ -1,8 +1,15 @@ import time +import os import subprocess + _ELASTIC_BUFFER_WRITES_TIME_IN_SEC: int = 1 +USE_STATIC_SLEEP: bool = bool(os.getenv("USE_STATIC_SLEEP", False)) +ELASTICSEARCH_REFRESH_INTERVAL_SECONDS: int = int(os.getenv("ELASTICSEARCH_REFRESH_INTERVAL_SECONDS", 5)) def wait_for_writes_to_sync(max_timeout_in_sec: int = 120) -> None: + if USE_STATIC_SLEEP: + time.sleep(ELASTICSEARCH_REFRESH_INTERVAL_SECONDS) + return start_time = time.time() # get offsets lag_zero = False From ae1b6328f77da4e8c6b621262c825f227ba97232 Mon Sep 17 00:00:00 2001 From: Joshua Eilers Date: Mon, 19 Jun 2023 04:08:55 -0700 Subject: [PATCH 033/222] chore(checkbox): darken all checkboxes (#8248) --- .../source/builder/RecipeForm/FormField.tsx | 13 +------------ .../src/conf/theme/global-overrides.less | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/FormField.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/FormField.tsx index d5f7f165981da..47092b078b3e9 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/FormField.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/FormField.tsx @@ -19,17 +19,6 @@ const StyledRemoveIcon = styled(MinusCircleOutlined)` margin-left: 10px; `; -const StyledCheckbox = styled(Checkbox)` - .ant-checkbox-inner { - border-color: ${ANTD_GRAY[7]}; - } - .ant-checkbox-checked { - .ant-checkbox-inner { - border-color: ${(props) => props.theme.styles['primary-color']}; - } - } -`; - interface CommonFieldProps { field: RecipeField; removeMargin?: boolean; @@ -132,7 +121,7 @@ function FormField(props: Props) { const isBoolean = field.type === FieldType.BOOLEAN; let input = ; - if (isBoolean) input = ; + if (isBoolean) input = ; if (field.type === FieldType.TEXTAREA) input = ; const valuePropName = isBoolean ? 'checked' : 'value'; diff --git a/datahub-web-react/src/conf/theme/global-overrides.less b/datahub-web-react/src/conf/theme/global-overrides.less index ec5ce1bdaae64..96e8937dbf19d 100644 --- a/datahub-web-react/src/conf/theme/global-overrides.less +++ b/datahub-web-react/src/conf/theme/global-overrides.less @@ -36,3 +36,18 @@ body { border-color: @gray-5; } } + +.ant-checkbox { + &:not(.ant-checkbox-disabled) { + &.ant-checkbox-checked { + .ant-checkbox-inner { + border-color: @primary-color; + } + } + &:not(.ant-checkbox-checked) { + .ant-checkbox-inner { + border-color: @gray-7; + } + } + } +} From abf73b2d10b3bc00325c243fa0acaf303cb74310 Mon Sep 17 00:00:00 2001 From: Joshua Eilers Date: Tue, 20 Jun 2023 03:06:44 -0700 Subject: [PATCH 034/222] chore(assertions): catch any exception on assertion delete (#8247) --- .../graphql/resolvers/assertion/DeleteAssertionResolver.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/DeleteAssertionResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/DeleteAssertionResolver.java index c4aaef5cc74fe..1d720535d3395 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/DeleteAssertionResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/DeleteAssertionResolver.java @@ -14,7 +14,6 @@ import com.linkedin.metadata.Constants; import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; import java.util.concurrent.CompletableFuture; @@ -54,7 +53,7 @@ public CompletableFuture get(final DataFetchingEnvironment environment) CompletableFuture.runAsync(() -> { try { _entityClient.deleteEntityReferences(assertionUrn, context.getAuthentication()); - } catch (RemoteInvocationException e) { + } catch (Exception e) { log.error(String.format("Caught exception while attempting to clear all entity references for assertion with urn %s", assertionUrn), e); } }); From 69d26e5b461119252c388b741e6efccb889edd01 Mon Sep 17 00:00:00 2001 From: Indy Prentice Date: Tue, 20 Jun 2023 16:54:10 -0500 Subject: [PATCH 035/222] feat(opensearch): Rollover usage events at a file size rather than time-based manner (#8182) Co-authored-by: Indy Prentice --- .../main/resources/index/usage-event/aws_es_ism_policy.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metadata-service/restli-servlet-impl/src/main/resources/index/usage-event/aws_es_ism_policy.json b/metadata-service/restli-servlet-impl/src/main/resources/index/usage-event/aws_es_ism_policy.json index bc1442d216918..7c5e41e9d4cf7 100644 --- a/metadata-service/restli-servlet-impl/src/main/resources/index/usage-event/aws_es_ism_policy.json +++ b/metadata-service/restli-servlet-impl/src/main/resources/index/usage-event/aws_es_ism_policy.json @@ -3,13 +3,14 @@ "policy_id": "PREFIXdatahub_usage_event_policy", "description": "Datahub Usage Event Policy", "default_state": "Rollover", - "schema_version": 1, + "schema_version": 2, "states": [ { "name": "Rollover", "actions": [ { "rollover": { + "min_size": "5gb", "min_index_age": "1d" } } From 02bc71dd84902133d00db1b17fd8a1c7ae74c6b6 Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Wed, 21 Jun 2023 04:08:59 -0400 Subject: [PATCH 036/222] fix(ingest/okta): Set default of okta_profile_to_username_attr to email (#8263) --- docs/how/updating-datahub.md | 3 +++ .../src/datahub/ingestion/source/identity/okta.py | 2 +- metadata-ingestion/tests/integration/okta/test_okta.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 59cc1348cb4cb..1368a57b5bee2 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -7,6 +7,9 @@ This file documents any backwards-incompatible changes in DataHub and assists pe ### Breaking Changes - #8201: Python SDK: In the DataFlow class, the `cluster` argument is deprecated in favor of `env`. +- #8263: Okta source config option `okta_profile_to_username_attr` default changed from `login` to `email`. + This determines which Okta profile attribute is used for the corresponding DataHub user + and thus may change what DataHub users are generated by the Okta source. ### Potential Downtime diff --git a/metadata-ingestion/src/datahub/ingestion/source/identity/okta.py b/metadata-ingestion/src/datahub/ingestion/source/identity/okta.py index 66ec9d71d44d4..ef41035ae0d1f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/identity/okta.py +++ b/metadata-ingestion/src/datahub/ingestion/source/identity/okta.py @@ -78,7 +78,7 @@ class OktaConfig(StatefulIngestionConfigBase, ConfigModel): # Optional: Customize the mapping to DataHub Username from an attribute appearing in the Okta User # profile. Reference: https://developer.okta.com/docs/reference/api/users/ okta_profile_to_username_attr: str = Field( - default="login", + default="email", description="Which Okta User Profile attribute to use as input to DataHub username mapping. Common values used are - login, email.", ) okta_profile_to_username_regex: str = Field( diff --git a/metadata-ingestion/tests/integration/okta/test_okta.py b/metadata-ingestion/tests/integration/okta/test_okta.py index 926100ae5be4e..c4f0c9000cb86 100644 --- a/metadata-ingestion/tests/integration/okta/test_okta.py +++ b/metadata-ingestion/tests/integration/okta/test_okta.py @@ -91,7 +91,7 @@ def test_okta_config(): assert config.ingest_users is True assert config.ingest_groups is True assert config.ingest_group_membership is True - assert config.okta_profile_to_username_attr == "login" + assert config.okta_profile_to_username_attr == "email" assert config.okta_profile_to_username_regex == "([^@]+)" assert config.okta_profile_to_group_name_attr == "name" assert config.okta_profile_to_group_name_regex == "(.*)" From ed2159bc3638d76bb35bdfd6f2f9e6ffb4368233 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Wed, 21 Jun 2023 04:10:07 -0400 Subject: [PATCH 037/222] feat(ui) Update Search & Browse to be a unified experience (#8235) Co-authored-by: Josh Eilers --- .../linkedin/datahub/graphql/Constants.java | 1 + .../datahub/graphql/GmsGraphQLEngine.java | 12 + .../graphql/featureflags/FeatureFlags.java | 2 + .../resolvers/chart/BrowseV2Resolver.java | 110 +++++ .../resolvers/config/AppConfigResolver.java | 8 +- .../graphql/types/chart/ChartType.java | 3 +- .../types/chart/mappers/ChartMapper.java | 4 + .../common/mappers/BrowsePathsV2Mapper.java | 36 ++ .../types/dashboard/DashboardType.java | 3 +- .../dashboard/mappers/DashboardMapper.java | 4 + .../graphql/types/dataflow/DataFlowType.java | 3 +- .../dataflow/mappers/DataFlowMapper.java | 4 + .../graphql/types/datajob/DataJobType.java | 3 +- .../types/datajob/mappers/DataJobMapper.java | 4 + .../graphql/types/dataset/DatasetType.java | 3 +- .../types/dataset/mappers/DatasetMapper.java | 4 + .../mlmodel/mappers/MLFeatureMapper.java | 4 + .../mlmodel/mappers/MLFeatureTableMapper.java | 4 + .../mlmodel/mappers/MLModelGroupMapper.java | 4 + .../types/mlmodel/mappers/MLModelMapper.java | 4 + .../graphql/types/notebook/NotebookType.java | 3 +- .../notebook/mappers/NotebookMapper.java | 4 + .../src/main/resources/app.graphql | 10 + .../src/main/resources/entity.graphql | 50 ++ .../src/main/resources/search.graphql | 126 +++++ .../browse/BrowseV2ResolverTest.java | 271 +++++++++++ datahub-web-react/src/Mocks.tsx | 48 +- datahub-web-react/src/app/analytics/event.ts | 71 ++- .../src/app/entity/EntityRegistry.tsx | 4 + .../shared/components/styled/StyledTag.tsx | 2 +- .../profile/nav/EntityProfileNavBar.tsx | 18 +- .../profile/nav/ProfileNavBrowsePath.tsx | 4 +- .../profile/nav/ProfileNavBrowsePathV2.tsx | 114 +++++ .../nav/useHasMultipleEnvironmentsQuery.ts | 17 + .../src/app/entity/shared/types.ts | 4 + .../app/entity/view/builder/ViewBuilder.tsx | 12 +- .../entity/view/builder/ViewBuilderForm.tsx | 4 +- .../entity/view/builder/ViewBuilderModal.tsx | 1 + .../view/builder/__tests__/utils.test.ts | 23 + .../src/app/entity/view/builder/utils.ts | 44 +- .../src/app/onboarding/OnboardingStep.tsx | 4 + .../src/app/onboarding/OnboardingTour.tsx | 7 +- .../config/SearchOnboardingConfig.tsx | 33 +- .../src/app/onboarding/utils.tsx | 51 ++- .../search/AdvancedSearchAddFilterSelect.tsx | 13 +- .../src/app/search/AdvancedSearchFilter.tsx | 85 ++-- .../AdvancedSearchFilterConditionSelect.tsx | 4 +- .../AdvancedSearchFilterValuesSection.tsx | 13 +- .../src/app/search/AdvancedSearchFilters.tsx | 27 +- .../src/app/search/BrowseEntityCard.tsx | 26 +- .../src/app/search/SearchBar.tsx | 1 + .../src/app/search/SearchFilterLabel.tsx | 9 +- .../src/app/search/SearchPage.tsx | 105 +++-- .../src/app/search/SearchResultList.tsx | 20 +- .../src/app/search/SearchResults.tsx | 119 +++-- .../src/app/search/ToggleSidebarButton.tsx | 21 + .../app/search/__tests__/SearchPage.test.tsx | 12 +- .../advanced/AdvancedFilterCloseButton.tsx | 31 ++ .../EntitySubTypeAdvancedFilterLabel.tsx | 105 +++++ .../src/app/search/advanced/styles.ts | 20 + .../src/app/search/filters/ActiveFilter.tsx | 109 +++++ .../app/search/filters/AdvancedFilters.tsx | 114 +++++ .../src/app/search/filters/BasicFilters.tsx | 139 ++++++ .../EntityTypeFilter/EntityTypeFilter.tsx | 91 ++++ .../EntityTypeFilter/entityTypeFilterUtils.ts | 114 +++++ .../src/app/search/filters/FilterOption.tsx | 187 ++++++++ .../app/search/filters/MoreFilterOption.tsx | 89 ++++ .../src/app/search/filters/MoreFilters.tsx | 74 +++ .../search/filters/OptionsDropdownMenu.tsx | 109 +++++ .../src/app/search/filters/ParentNodes.tsx | 87 ++++ .../src/app/search/filters/SaveViewButton.tsx | 69 +++ .../src/app/search/filters/SearchFilter.tsx | 50 ++ .../app/search/filters/SearchFilterView.tsx | 90 ++++ .../src/app/search/filters/SearchFilters.tsx | 64 +++ .../__tests__/entityTypeFilterUtils.test.tsx | 273 +++++++++++ .../search/filters/__tests__/utils.test.tsx | 431 ++++++++++++++++++ .../src/app/search/filters/constants.ts | 32 ++ .../app/search/filters/mapFilterOption.tsx | 51 +++ .../app/search/filters/styledComponents.ts | 13 + .../src/app/search/filters/types.ts | 8 + .../src/app/search/filters/useFilterMode.ts | 16 + .../filters/useGetBrowseV2LabelOverride.ts | 32 ++ .../filters/useSearchFilterAnalytics.ts | 23 + .../filters/useSearchFilterDropdown.tsx | 125 +++++ .../src/app/search/filters/utils.tsx | 333 ++++++++++++++ .../src/app/search/sidebar/BrowseContext.tsx | 222 +++++++++ .../src/app/search/sidebar/BrowseNode.tsx | 118 +++++ .../src/app/search/sidebar/BrowseSidebar.tsx | 66 +++ .../src/app/search/sidebar/EntityNode.tsx | 103 +++++ .../app/search/sidebar/EnvironmentNode.tsx | 84 ++++ .../src/app/search/sidebar/ExpandableNode.tsx | 163 +++++++ .../src/app/search/sidebar/PlatformNode.tsx | 115 +++++ .../src/app/search/sidebar/SidebarContext.tsx | 63 +++ .../search/sidebar/SidebarLoadingError.tsx | 25 + .../__tests__/browseContextUtils.test.ts | 20 + .../app/search/sidebar/browseContextUtils.ts | 8 + .../src/app/search/sidebar/constants.ts | 4 + .../src/app/search/sidebar/index.tsx | 3 + .../src/app/search/sidebar/types.ts | 6 + .../search/sidebar/useAggregationsQuery.ts | 72 +++ .../search/sidebar/useBrowsePagination.tsx | 100 ++++ .../app/search/sidebar/useSidebarAnalytics.ts | 54 +++ .../app/search/sidebar/useSidebarFilters.ts | 59 +++ .../search/useAdvancedSearchSelectFilters.tsx | 31 ++ .../src/app/search/useGetSearchQueryInputs.ts | 49 ++ .../app/search/useSearchAndBrowseVersion.ts | 29 ++ .../__tests__/applyFilterOverrides.test.ts | 105 +++++ .../utils/__tests__/generateOrFilters.test.ts | 90 ++++ .../app/search/utils/applyFilterOverrides.ts | 31 ++ .../src/app/search/utils/constants.ts | 23 + .../src/app/search/utils/generateOrFilters.ts | 41 +- .../src/app/shared/useIntersect.ts | 35 ++ datahub-web-react/src/app/shared/useLatest.ts | 11 + datahub-web-react/src/app/shared/useToggle.ts | 26 ++ .../src/app/useEntityRegistry.ts | 2 - datahub-web-react/src/appConfigContext.tsx | 2 + datahub-web-react/src/graphql/app.graphql | 2 + .../src/graphql/browseV2.graphql | 40 ++ datahub-web-react/src/graphql/chart.graphql | 3 + .../src/graphql/dataFlow.graphql | 3 + .../src/graphql/fragments.graphql | 164 +++++++ datahub-web-react/src/graphql/search.graphql | 14 + datahub-web-react/src/images/collapse.svg | 4 + datahub-web-react/src/images/domain.svg | 3 + datahub-web-react/src/images/expand.svg | 4 + .../utils/test-utils/TestPageContainer.tsx | 47 +- docker/docker-compose.dev.yml | 1 + .../java/com/linkedin/metadata/Constants.java | 1 + .../metadata/client/JavaEntityClient.java | 31 +- .../datahubusage/DataHubUsageEventType.java | 5 + .../metadata/entity/EntityService.java | 28 ++ .../metadata/search/EntitySearchService.java | 15 + .../AllEntitiesSearchAggregator.java | 16 +- .../elasticsearch/ElasticSearchService.java | 13 +- .../indexbuilder/SettingsBuilder.java | 12 +- .../elasticsearch/query/ESBrowseDAO.java | 165 +++++++ .../elasticsearch/query/ESSearchDAO.java | 45 -- .../query/request/SearchFieldConfig.java | 1 + .../query/request/SearchRequestHandler.java | 17 +- .../search/utils/BrowsePathV2Utils.java | 162 +++++++ .../metadata/search/utils/ESUtils.java | 2 + .../linkedin/metadata/AspectUtilsTest.java | 3 +- .../metadata/ESSampleDataFixture.java | 5 +- .../metadata/ESSearchLineageFixture.java | 6 +- .../metadata/ESTestConfiguration.java | 11 + .../search/LineageSearchServiceTest.java | 5 +- .../metadata/search/SearchServiceTest.java | 5 +- .../ElasticSearchServiceTest.java | 5 +- .../elasticsearch/query/ESBrowseDAOTest.java | 23 +- .../elasticsearch/query/ESSearchDAOTest.java | 10 +- .../search/utils/BrowsePathV2UtilsTest.java | 201 ++++++++ .../metadata/browse/BrowseResultGroupV2.pdl | 29 ++ .../metadata/browse/BrowseResultV2.pdl | 32 ++ .../linkedin/metadata/query/SearchFlags.pdl | 2 +- .../search/ElasticSearchServiceFactory.java | 3 +- .../factories/BootstrapManagerFactory.java | 13 + .../boot/steps/BackfillBrowsePathsV2Step.java | 111 +++++ .../src/main/resources/application.yml | 4 + .../steps/BackfillBrowsePathsV2StepTest.java | 171 +++++++ .../linkedin/entity/client/EntityClient.java | 17 + .../entity/client/RestliEntityClient.java | 19 + .../linkedin/metadata/utils/SearchUtil.java | 75 ++- .../cypress/cypress/e2e/browse/browseV2.js | 147 ++++++ .../cypress/cypress/e2e/mutations/domains.js | 4 +- .../cypress/e2e/search/searchFilters.js | 93 ++++ smoke-test/tests/cypress/cypress/e2e/utils.js | 13 + smoke-test/tests/cypress/integration_test.py | 2 + 167 files changed, 7722 insertions(+), 324 deletions(-) create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/BrowsePathsV2Mapper.java create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseV2ResolverTest.java create mode 100644 datahub-web-react/src/app/entity/shared/containers/profile/nav/ProfileNavBrowsePathV2.tsx create mode 100644 datahub-web-react/src/app/entity/shared/containers/profile/nav/useHasMultipleEnvironmentsQuery.ts create mode 100644 datahub-web-react/src/app/entity/view/builder/__tests__/utils.test.ts create mode 100644 datahub-web-react/src/app/search/ToggleSidebarButton.tsx create mode 100644 datahub-web-react/src/app/search/advanced/AdvancedFilterCloseButton.tsx create mode 100644 datahub-web-react/src/app/search/advanced/EntitySubTypeAdvancedFilterLabel.tsx create mode 100644 datahub-web-react/src/app/search/advanced/styles.ts create mode 100644 datahub-web-react/src/app/search/filters/ActiveFilter.tsx create mode 100644 datahub-web-react/src/app/search/filters/AdvancedFilters.tsx create mode 100644 datahub-web-react/src/app/search/filters/BasicFilters.tsx create mode 100644 datahub-web-react/src/app/search/filters/EntityTypeFilter/EntityTypeFilter.tsx create mode 100644 datahub-web-react/src/app/search/filters/EntityTypeFilter/entityTypeFilterUtils.ts create mode 100644 datahub-web-react/src/app/search/filters/FilterOption.tsx create mode 100644 datahub-web-react/src/app/search/filters/MoreFilterOption.tsx create mode 100644 datahub-web-react/src/app/search/filters/MoreFilters.tsx create mode 100644 datahub-web-react/src/app/search/filters/OptionsDropdownMenu.tsx create mode 100644 datahub-web-react/src/app/search/filters/ParentNodes.tsx create mode 100644 datahub-web-react/src/app/search/filters/SaveViewButton.tsx create mode 100644 datahub-web-react/src/app/search/filters/SearchFilter.tsx create mode 100644 datahub-web-react/src/app/search/filters/SearchFilterView.tsx create mode 100644 datahub-web-react/src/app/search/filters/SearchFilters.tsx create mode 100644 datahub-web-react/src/app/search/filters/__tests__/entityTypeFilterUtils.test.tsx create mode 100644 datahub-web-react/src/app/search/filters/__tests__/utils.test.tsx create mode 100644 datahub-web-react/src/app/search/filters/constants.ts create mode 100644 datahub-web-react/src/app/search/filters/mapFilterOption.tsx create mode 100644 datahub-web-react/src/app/search/filters/styledComponents.ts create mode 100644 datahub-web-react/src/app/search/filters/types.ts create mode 100644 datahub-web-react/src/app/search/filters/useFilterMode.ts create mode 100644 datahub-web-react/src/app/search/filters/useGetBrowseV2LabelOverride.ts create mode 100644 datahub-web-react/src/app/search/filters/useSearchFilterAnalytics.ts create mode 100644 datahub-web-react/src/app/search/filters/useSearchFilterDropdown.tsx create mode 100644 datahub-web-react/src/app/search/filters/utils.tsx create mode 100644 datahub-web-react/src/app/search/sidebar/BrowseContext.tsx create mode 100644 datahub-web-react/src/app/search/sidebar/BrowseNode.tsx create mode 100644 datahub-web-react/src/app/search/sidebar/BrowseSidebar.tsx create mode 100644 datahub-web-react/src/app/search/sidebar/EntityNode.tsx create mode 100644 datahub-web-react/src/app/search/sidebar/EnvironmentNode.tsx create mode 100644 datahub-web-react/src/app/search/sidebar/ExpandableNode.tsx create mode 100644 datahub-web-react/src/app/search/sidebar/PlatformNode.tsx create mode 100644 datahub-web-react/src/app/search/sidebar/SidebarContext.tsx create mode 100644 datahub-web-react/src/app/search/sidebar/SidebarLoadingError.tsx create mode 100644 datahub-web-react/src/app/search/sidebar/__tests__/browseContextUtils.test.ts create mode 100644 datahub-web-react/src/app/search/sidebar/browseContextUtils.ts create mode 100644 datahub-web-react/src/app/search/sidebar/constants.ts create mode 100644 datahub-web-react/src/app/search/sidebar/index.tsx create mode 100644 datahub-web-react/src/app/search/sidebar/types.ts create mode 100644 datahub-web-react/src/app/search/sidebar/useAggregationsQuery.ts create mode 100644 datahub-web-react/src/app/search/sidebar/useBrowsePagination.tsx create mode 100644 datahub-web-react/src/app/search/sidebar/useSidebarAnalytics.ts create mode 100644 datahub-web-react/src/app/search/sidebar/useSidebarFilters.ts create mode 100644 datahub-web-react/src/app/search/useAdvancedSearchSelectFilters.tsx create mode 100644 datahub-web-react/src/app/search/useGetSearchQueryInputs.ts create mode 100644 datahub-web-react/src/app/search/useSearchAndBrowseVersion.ts create mode 100644 datahub-web-react/src/app/search/utils/__tests__/applyFilterOverrides.test.ts create mode 100644 datahub-web-react/src/app/search/utils/__tests__/generateOrFilters.test.ts create mode 100644 datahub-web-react/src/app/search/utils/applyFilterOverrides.ts create mode 100644 datahub-web-react/src/app/shared/useIntersect.ts create mode 100644 datahub-web-react/src/app/shared/useLatest.ts create mode 100644 datahub-web-react/src/app/shared/useToggle.ts create mode 100644 datahub-web-react/src/graphql/browseV2.graphql create mode 100644 datahub-web-react/src/images/collapse.svg create mode 100644 datahub-web-react/src/images/domain.svg create mode 100644 datahub-web-react/src/images/expand.svg create mode 100644 metadata-io/src/main/java/com/linkedin/metadata/search/utils/BrowsePathV2Utils.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/search/utils/BrowsePathV2UtilsTest.java create mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/browse/BrowseResultGroupV2.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/browse/BrowseResultV2.pdl create mode 100644 metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java create mode 100644 metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2StepTest.java create mode 100644 smoke-test/tests/cypress/cypress/e2e/browse/browseV2.js create mode 100644 smoke-test/tests/cypress/cypress/e2e/search/searchFilters.js create mode 100644 smoke-test/tests/cypress/cypress/e2e/utils.js diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java index a70d1a1b62c7e..4488f27c19d80 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java @@ -21,6 +21,7 @@ public class Constants { public static final String STEPS_SCHEMA_FILE = "step.graphql"; public static final String LINEAGE_SCHEMA_FILE = "lineage.graphql"; public static final String BROWSE_PATH_DELIMITER = "/"; + public static final String BROWSE_PATH_V2_DELIMITER = "␟"; public static final String VERSION_STAMP_FIELD_NAME = "versionStamp"; public static final String ENTITY_FILTER_NAME = "_entityType"; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index d3fbfafe5e5dd..81279cac27ce0 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -26,6 +26,8 @@ import com.linkedin.datahub.graphql.generated.Assertion; import com.linkedin.datahub.graphql.generated.AutoCompleteResultForEntity; import com.linkedin.datahub.graphql.generated.AutoCompleteResults; +import com.linkedin.datahub.graphql.generated.BrowsePathEntry; +import com.linkedin.datahub.graphql.generated.BrowseResultGroupV2; import com.linkedin.datahub.graphql.generated.BrowseResults; import com.linkedin.datahub.graphql.generated.Chart; import com.linkedin.datahub.graphql.generated.ChartInfo; @@ -101,6 +103,7 @@ import com.linkedin.datahub.graphql.resolvers.browse.BrowsePathsResolver; import com.linkedin.datahub.graphql.resolvers.browse.BrowseResolver; import com.linkedin.datahub.graphql.resolvers.browse.EntityBrowsePathsResolver; +import com.linkedin.datahub.graphql.resolvers.chart.BrowseV2Resolver; import com.linkedin.datahub.graphql.resolvers.chart.ChartStatsSummaryResolver; import com.linkedin.datahub.graphql.resolvers.config.AppConfigResolver; import com.linkedin.datahub.graphql.resolvers.container.ContainerEntitiesResolver; @@ -780,6 +783,7 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("dataProduct", getResolver(dataProductType)) .dataFetcher("listDataProductAssets", new ListDataProductAssetsResolver(this.entityClient)) .dataFetcher("listOwnershipTypes", new ListOwnershipTypesResolver(this.entityClient)) + .dataFetcher("browseV2", new BrowseV2Resolver(this.entityClient, this.viewService)) ); } @@ -955,6 +959,14 @@ private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder .dataFetcher("entity", new EntityTypeResolver(entityTypes, (env) -> ((EntityRelationship) env.getSource()).getEntity())) ) + .type("BrowseResultGroupV2", typeWiring -> typeWiring + .dataFetcher("entity", new EntityTypeResolver(entityTypes, + (env) -> ((BrowseResultGroupV2) env.getSource()).getEntity())) + ) + .type("BrowsePathEntry", typeWiring -> typeWiring + .dataFetcher("entity", new EntityTypeResolver(entityTypes, + (env) -> ((BrowsePathEntry) env.getSource()).getEntity())) + ) .type("LineageRelationship", typeWiring -> typeWiring .dataFetcher("entity", new EntityTypeResolver(entityTypes, (env) -> ((LineageRelationship) env.getSource()).getEntity())) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java index 0287e6656a229..49af7265ec9d7 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java @@ -11,5 +11,7 @@ public class FeatureFlags { private boolean pointInTimeCreationEnabled = false; private boolean alwaysEmitChangeLog = false; private boolean readOnlyModeEnabled = false; + private boolean showSearchFiltersV2 = false; + private boolean showBrowseV2 = false; private PreProcessHooks preProcessHooks; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java new file mode 100644 index 0000000000000..41a1d22485ea4 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java @@ -0,0 +1,110 @@ +package com.linkedin.datahub.graphql.resolvers.chart; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.BrowseResultGroupV2; +import com.linkedin.datahub.graphql.generated.BrowseResultMetadata; +import com.linkedin.datahub.graphql.generated.BrowseResultsV2; +import com.linkedin.datahub.graphql.generated.BrowseV2Input; +import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; +import com.linkedin.datahub.graphql.resolvers.ResolverUtils; +import com.linkedin.datahub.graphql.resolvers.search.SearchUtils; +import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.browse.BrowseResultV2; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.service.ViewService; +import com.linkedin.view.DataHubViewInfo; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; + +import static com.linkedin.datahub.graphql.Constants.BROWSE_PATH_V2_DELIMITER; +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; +import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.resolveView; + +@Slf4j +@RequiredArgsConstructor +public class BrowseV2Resolver implements DataFetcher> { + + private final EntityClient _entityClient; + private final ViewService _viewService; + + private static final int DEFAULT_START = 0; + private static final int DEFAULT_COUNT = 10; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) { + final QueryContext context = environment.getContext(); + final BrowseV2Input input = bindArgument(environment.getArgument("input"), BrowseV2Input.class); + final String entityName = EntityTypeMapper.getName(input.getType()); + + final int start = input.getStart() != null ? input.getStart() : DEFAULT_START; + final int count = input.getCount() != null ? input.getCount() : DEFAULT_COUNT; + final String query = input.getQuery() != null ? input.getQuery() : "*"; + + return CompletableFuture.supplyAsync(() -> { + try { + final DataHubViewInfo maybeResolvedView = (input.getViewUrn() != null) + ? resolveView(_viewService, UrnUtils.getUrn(input.getViewUrn()), context.getAuthentication()) + : null; + final String pathStr = input.getPath().size() > 0 ? BROWSE_PATH_V2_DELIMITER + String.join(BROWSE_PATH_V2_DELIMITER, input.getPath()) : ""; + final Filter filter = ResolverUtils.buildFilter(null, input.getOrFilters()); + + BrowseResultV2 browseResults = _entityClient.browseV2( + entityName, + pathStr, + maybeResolvedView != null + ? SearchUtils.combineFilters(filter, maybeResolvedView.getDefinition().getFilter()) + : filter, + query, + start, + count, + context.getAuthentication() + ); + return mapBrowseResults(browseResults); + } catch (Exception e) { + throw new RuntimeException("Failed to execute browse V2", e); + } + }); + } + + private BrowseResultsV2 mapBrowseResults(BrowseResultV2 browseResults) { + BrowseResultsV2 results = new BrowseResultsV2(); + results.setTotal(browseResults.getNumGroups()); + results.setStart(browseResults.getFrom()); + results.setCount(browseResults.getPageSize()); + + List groups = new ArrayList<>(); + browseResults.getGroups().forEach(group -> { + BrowseResultGroupV2 browseGroup = new BrowseResultGroupV2(); + browseGroup.setName(group.getName()); + browseGroup.setCount(group.getCount()); + browseGroup.setHasSubGroups(group.isHasSubGroups()); + if (group.hasUrn() && group.getUrn() != null) { + browseGroup.setEntity(UrnToEntityMapper.map(group.getUrn())); + } + groups.add(browseGroup); + }); + results.setGroups(groups); + + BrowseResultMetadata resultMetadata = new BrowseResultMetadata(); + resultMetadata.setPath(Arrays.stream(browseResults.getMetadata().getPath() + .split(BROWSE_PATH_V2_DELIMITER)) + .filter(pathComponent -> !"".equals(pathComponent)) + .collect(Collectors.toList()) + ); + resultMetadata.setTotalNumEntities(browseResults.getMetadata().getTotalNumEntities()); + results.setMetadata(resultMetadata); + + return results; + } +} + diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java index 4c12aaaee47a0..48f0617c9975d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java @@ -147,8 +147,12 @@ public CompletableFuture get(final DataFetchingEnvironment environmen viewsConfig.setEnabled(_viewsConfiguration.isEnabled()); appConfig.setViewsConfig(viewsConfig); - final FeatureFlagsConfig featureFlagsConfig = new FeatureFlagsConfig(); - featureFlagsConfig.setReadOnlyModeEnabled(_featureFlags.isReadOnlyModeEnabled()); + final FeatureFlagsConfig featureFlagsConfig = FeatureFlagsConfig.builder() + .setShowSearchFiltersV2(_featureFlags.isShowSearchFiltersV2()) + .setReadOnlyModeEnabled(_featureFlags.isReadOnlyModeEnabled()) + .setShowBrowseV2(_featureFlags.isShowBrowseV2()) + .build(); + appConfig.setFeatureFlags(featureFlagsConfig); return CompletableFuture.completedFuture(appConfig); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java index 35b1c43ba2bc1..cfec8f8a2391f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java @@ -76,7 +76,8 @@ public class ChartType implements SearchableEntityType, Browsable DATA_PLATFORM_INSTANCE_ASPECT_NAME, INPUT_FIELDS_ASPECT_NAME, EMBED_ASPECT_NAME, - DATA_PRODUCTS_ASPECT_NAME + DATA_PRODUCTS_ASPECT_NAME, + BROWSE_PATHS_V2_ASPECT_NAME ); private static final Set FACET_FIELDS = ImmutableSet.of("access", "queryType", "tool", "type"); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/ChartMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/ChartMapper.java index 6aebf3f9bb7ef..8b525dd587753 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/ChartMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/ChartMapper.java @@ -1,6 +1,7 @@ package com.linkedin.datahub.graphql.types.chart.mappers; import com.linkedin.chart.EditableChartProperties; +import com.linkedin.common.BrowsePathsV2; import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.Deprecation; import com.linkedin.common.Embed; @@ -25,6 +26,7 @@ import com.linkedin.datahub.graphql.generated.Dataset; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.types.common.mappers.AuditStampMapper; +import com.linkedin.datahub.graphql.types.common.mappers.BrowsePathsV2Mapper; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper; import com.linkedin.datahub.graphql.types.common.mappers.EmbedMapper; @@ -93,6 +95,8 @@ public Chart apply(@Nonnull final EntityResponse entityResponse) { chart.setInputFields(InputFieldsMapper.map(new InputFields(dataMap), entityUrn))); mappingHelper.mapToResult(EMBED_ASPECT_NAME, (chart, dataMap) -> chart.setEmbed(EmbedMapper.map(new Embed(dataMap)))); + mappingHelper.mapToResult(BROWSE_PATHS_V2_ASPECT_NAME, (chart, dataMap) -> + chart.setBrowsePathV2(BrowsePathsV2Mapper.map(new BrowsePathsV2(dataMap)))); return mappingHelper.getResult(); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/BrowsePathsV2Mapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/BrowsePathsV2Mapper.java new file mode 100644 index 0000000000000..41ee99fa412ad --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/BrowsePathsV2Mapper.java @@ -0,0 +1,36 @@ +package com.linkedin.datahub.graphql.types.common.mappers; + +import com.linkedin.common.BrowsePathsV2; +import com.linkedin.datahub.graphql.generated.BrowsePathEntry; +import com.linkedin.datahub.graphql.generated.BrowsePathV2; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; + +import javax.annotation.Nonnull; +import java.util.List; +import java.util.stream.Collectors; + +public class BrowsePathsV2Mapper implements ModelMapper { + + public static final BrowsePathsV2Mapper INSTANCE = new BrowsePathsV2Mapper(); + + public static BrowsePathV2 map(@Nonnull final BrowsePathsV2 metadata) { + return INSTANCE.apply(metadata); + } + + @Override + public BrowsePathV2 apply(@Nonnull final BrowsePathsV2 input) { + final BrowsePathV2 result = new BrowsePathV2(); + final List path = input.getPath().stream().map(this::mapBrowsePathEntry).collect(Collectors.toList()); + result.setPath(path); + return result; + } + + private BrowsePathEntry mapBrowsePathEntry(com.linkedin.common.BrowsePathEntry pathEntry) { + final BrowsePathEntry entry = new BrowsePathEntry(); + entry.setName(pathEntry.getId()); + if (pathEntry.hasUrn() && pathEntry.getUrn() != null) { + entry.setEntity(UrnToEntityMapper.map(pathEntry.getUrn())); + } + return entry; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java index fb512a4b52e89..104c7c004cb66 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java @@ -78,7 +78,8 @@ public class DashboardType implements SearchableEntityType, B INPUT_FIELDS_ASPECT_NAME, SUB_TYPES_ASPECT_NAME, EMBED_ASPECT_NAME, - DATA_PRODUCTS_ASPECT_NAME + DATA_PRODUCTS_ASPECT_NAME, + BROWSE_PATHS_V2_ASPECT_NAME ); private static final Set FACET_FIELDS = ImmutableSet.of("access", "tool"); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java index 709a1678eefda..d11767c7bf41d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql.types.dashboard.mappers; +import com.linkedin.common.BrowsePathsV2; import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.Deprecation; import com.linkedin.common.Embed; @@ -24,6 +25,7 @@ import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.types.chart.mappers.InputFieldsMapper; import com.linkedin.datahub.graphql.types.common.mappers.AuditStampMapper; +import com.linkedin.datahub.graphql.types.common.mappers.BrowsePathsV2Mapper; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper; import com.linkedin.datahub.graphql.types.common.mappers.EmbedMapper; @@ -92,6 +94,8 @@ public Dashboard apply(@Nonnull final EntityResponse entityResponse) { mappingHelper.mapToResult(SUB_TYPES_ASPECT_NAME, this::mapSubTypes); mappingHelper.mapToResult(EMBED_ASPECT_NAME, (dashboard, dataMap) -> dashboard.setEmbed(EmbedMapper.map(new Embed(dataMap)))); + mappingHelper.mapToResult(BROWSE_PATHS_V2_ASPECT_NAME, (dashboard, dataMap) -> + dashboard.setBrowsePathV2(BrowsePathsV2Mapper.map(new BrowsePathsV2(dataMap)))); return mappingHelper.getResult(); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java index adcc7feefb2e1..54f7660064c05 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java @@ -73,7 +73,8 @@ public class DataFlowType implements SearchableEntityType, Bro DOMAINS_ASPECT_NAME, DEPRECATION_ASPECT_NAME, DATA_PLATFORM_INSTANCE_ASPECT_NAME, - DATA_PRODUCTS_ASPECT_NAME + DATA_PRODUCTS_ASPECT_NAME, + BROWSE_PATHS_V2_ASPECT_NAME ); private static final Set FACET_FIELDS = ImmutableSet.of("orchestrator", "cluster"); private final EntityClient _entityClient; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java index 6ab2243c8eb1f..eec52facf2a63 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql.types.dataflow.mappers; +import com.linkedin.common.BrowsePathsV2; import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.Deprecation; import com.linkedin.common.GlobalTags; @@ -15,6 +16,7 @@ import com.linkedin.datahub.graphql.generated.DataFlowProperties; import com.linkedin.datahub.graphql.generated.DataPlatform; import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.types.common.mappers.BrowsePathsV2Mapper; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper; import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; @@ -76,6 +78,8 @@ public DataFlow apply(@Nonnull final EntityResponse entityResponse) { dataFlow.setDeprecation(DeprecationMapper.map(new Deprecation(dataMap)))); mappingHelper.mapToResult(DATA_PLATFORM_INSTANCE_ASPECT_NAME, (dataset, dataMap) -> dataset.setDataPlatformInstance(DataPlatformInstanceAspectMapper.map(new DataPlatformInstance(dataMap)))); + mappingHelper.mapToResult(BROWSE_PATHS_V2_ASPECT_NAME, (dataFlow, dataMap) -> + dataFlow.setBrowsePathV2(BrowsePathsV2Mapper.map(new BrowsePathsV2(dataMap)))); return mappingHelper.getResult(); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java index 8499a1af730b0..bde79f6dce6e8 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java @@ -74,7 +74,8 @@ public class DataJobType implements SearchableEntityType, Brows DOMAINS_ASPECT_NAME, DEPRECATION_ASPECT_NAME, DATA_PLATFORM_INSTANCE_ASPECT_NAME, - DATA_PRODUCTS_ASPECT_NAME + DATA_PRODUCTS_ASPECT_NAME, + BROWSE_PATHS_V2_ASPECT_NAME ); private static final Set FACET_FIELDS = ImmutableSet.of("flow"); private final EntityClient _entityClient; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java index c12ce67fc5095..e592ce2cdc2b5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java @@ -1,6 +1,7 @@ package com.linkedin.datahub.graphql.types.datajob.mappers; import com.google.common.collect.ImmutableList; +import com.linkedin.common.BrowsePathsV2; import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.Deprecation; import com.linkedin.common.GlobalTags; @@ -18,6 +19,7 @@ import com.linkedin.datahub.graphql.generated.DataJobProperties; import com.linkedin.datahub.graphql.generated.Dataset; import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.types.common.mappers.BrowsePathsV2Mapper; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper; import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; @@ -98,6 +100,8 @@ public DataJob apply(@Nonnull final EntityResponse entityResponse) { result.setDeprecation(DeprecationMapper.map(new Deprecation(data))); } else if (DATA_PLATFORM_INSTANCE_ASPECT_NAME.equals(name)) { result.setDataPlatformInstance(DataPlatformInstanceAspectMapper.map(new DataPlatformInstance(data))); + } else if (BROWSE_PATHS_V2_ASPECT_NAME.equals(name)) { + result.setBrowsePathV2(BrowsePathsV2Mapper.map(new BrowsePathsV2(data))); } }); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java index 963a609500d55..510971c8db11c 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java @@ -84,7 +84,8 @@ public class DatasetType implements SearchableEntityType, Brows DATA_PLATFORM_INSTANCE_ASPECT_NAME, SIBLINGS_ASPECT_NAME, EMBED_ASPECT_NAME, - DATA_PRODUCTS_ASPECT_NAME + DATA_PRODUCTS_ASPECT_NAME, + BROWSE_PATHS_V2_ASPECT_NAME ); private static final Set FACET_FIELDS = ImmutableSet.of("origin", "platform"); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java index deb86b4595c41..c5b810951d491 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql.types.dataset.mappers; +import com.linkedin.common.BrowsePathsV2; import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.Deprecation; import com.linkedin.common.Embed; @@ -18,6 +19,7 @@ import com.linkedin.datahub.graphql.generated.DatasetEditableProperties; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FabricType; +import com.linkedin.datahub.graphql.types.common.mappers.BrowsePathsV2Mapper; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper; import com.linkedin.datahub.graphql.types.common.mappers.EmbedMapper; @@ -106,6 +108,8 @@ public Dataset apply(@Nonnull final EntityResponse entityResponse) { dataset.setFineGrainedLineages(UpstreamLineagesMapper.map(new UpstreamLineage(dataMap)))); mappingHelper.mapToResult(EMBED_ASPECT_NAME, (dataset, dataMap) -> dataset.setEmbed(EmbedMapper.map(new Embed(dataMap)))); + mappingHelper.mapToResult(BROWSE_PATHS_V2_ASPECT_NAME, (dataset, dataMap) -> + dataset.setBrowsePathV2(BrowsePathsV2Mapper.map(new BrowsePathsV2(dataMap)))); return mappingHelper.getResult(); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java index f073bd71a1377..eb6551bafc726 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql.types.mlmodel.mappers; +import com.linkedin.common.BrowsePathsV2; import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.Deprecation; @@ -16,6 +17,7 @@ import com.linkedin.datahub.graphql.generated.MLFeature; import com.linkedin.datahub.graphql.generated.MLFeatureDataType; import com.linkedin.datahub.graphql.generated.MLFeatureEditableProperties; +import com.linkedin.datahub.graphql.types.common.mappers.BrowsePathsV2Mapper; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper; import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; @@ -80,6 +82,8 @@ public MLFeature apply(@Nonnull final EntityResponse entityResponse) { mappingHelper.mapToResult(ML_FEATURE_EDITABLE_PROPERTIES_ASPECT_NAME, this::mapEditableProperties); mappingHelper.mapToResult(DATA_PLATFORM_INSTANCE_ASPECT_NAME, (dataset, dataMap) -> dataset.setDataPlatformInstance(DataPlatformInstanceAspectMapper.map(new DataPlatformInstance(dataMap)))); + mappingHelper.mapToResult(BROWSE_PATHS_V2_ASPECT_NAME, (entity, dataMap) -> + entity.setBrowsePathV2(BrowsePathsV2Mapper.map(new BrowsePathsV2(dataMap)))); return mappingHelper.getResult(); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java index 88bf52e84da27..fa6b77d870757 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql.types.mlmodel.mappers; +import com.linkedin.common.BrowsePathsV2; import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.Deprecation; @@ -15,6 +16,7 @@ import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.MLFeatureTable; import com.linkedin.datahub.graphql.generated.MLFeatureTableEditableProperties; +import com.linkedin.datahub.graphql.types.common.mappers.BrowsePathsV2Mapper; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper; import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; @@ -79,6 +81,8 @@ public MLFeatureTable apply(@Nonnull final EntityResponse entityResponse) { mappingHelper.mapToResult(ML_FEATURE_TABLE_EDITABLE_PROPERTIES_ASPECT_NAME, this::mapEditableProperties); mappingHelper.mapToResult(DATA_PLATFORM_INSTANCE_ASPECT_NAME, (dataset, dataMap) -> dataset.setDataPlatformInstance(DataPlatformInstanceAspectMapper.map(new DataPlatformInstance(dataMap)))); + mappingHelper.mapToResult(BROWSE_PATHS_V2_ASPECT_NAME, (entity, dataMap) -> + entity.setBrowsePathV2(BrowsePathsV2Mapper.map(new BrowsePathsV2(dataMap)))); return mappingHelper.getResult(); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupMapper.java index 9460a0caea4d7..52db5468e0091 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupMapper.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql.types.mlmodel.mappers; +import com.linkedin.common.BrowsePathsV2; import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.Deprecation; import com.linkedin.common.GlobalTags; @@ -14,6 +15,7 @@ import com.linkedin.datahub.graphql.generated.FabricType; import com.linkedin.datahub.graphql.generated.MLModelGroup; import com.linkedin.datahub.graphql.generated.MLModelGroupEditableProperties; +import com.linkedin.datahub.graphql.types.common.mappers.BrowsePathsV2Mapper; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper; import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; @@ -75,6 +77,8 @@ public MLModelGroup apply(@Nonnull final EntityResponse entityResponse) { mappingHelper.mapToResult(ML_MODEL_GROUP_EDITABLE_PROPERTIES_ASPECT_NAME, this::mapEditableProperties); mappingHelper.mapToResult(DATA_PLATFORM_INSTANCE_ASPECT_NAME, (dataset, dataMap) -> dataset.setDataPlatformInstance(DataPlatformInstanceAspectMapper.map(new DataPlatformInstance(dataMap)))); + mappingHelper.mapToResult(BROWSE_PATHS_V2_ASPECT_NAME, (mlModelGroup, dataMap) -> + mlModelGroup.setBrowsePathV2(BrowsePathsV2Mapper.map(new BrowsePathsV2(dataMap)))); return mappingHelper.getResult(); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelMapper.java index 99f2d709765f1..ef38abc03c501 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelMapper.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql.types.mlmodel.mappers; +import com.linkedin.common.BrowsePathsV2; import com.linkedin.common.Cost; import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.Deprecation; @@ -16,6 +17,7 @@ import com.linkedin.datahub.graphql.generated.FabricType; import com.linkedin.datahub.graphql.generated.MLModel; import com.linkedin.datahub.graphql.generated.MLModelEditableProperties; +import com.linkedin.datahub.graphql.types.common.mappers.BrowsePathsV2Mapper; import com.linkedin.datahub.graphql.types.common.mappers.CostMapper; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper; @@ -113,6 +115,8 @@ public MLModel apply(@Nonnull final EntityResponse entityResponse) { mappingHelper.mapToResult(ML_MODEL_EDITABLE_PROPERTIES_ASPECT_NAME, this::mapEditableProperties); mappingHelper.mapToResult(DATA_PLATFORM_INSTANCE_ASPECT_NAME, (dataset, dataMap) -> dataset.setDataPlatformInstance(DataPlatformInstanceAspectMapper.map(new DataPlatformInstance(dataMap)))); + mappingHelper.mapToResult(BROWSE_PATHS_V2_ASPECT_NAME, (mlModel, dataMap) -> + mlModel.setBrowsePathV2(BrowsePathsV2Mapper.map(new BrowsePathsV2(dataMap)))); return mappingHelper.getResult(); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/NotebookType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/NotebookType.java index 246d54b2600e7..080cdeba09f19 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/NotebookType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/NotebookType.java @@ -70,7 +70,8 @@ public class NotebookType implements SearchableEntityType, Bro INSTITUTIONAL_MEMORY_ASPECT_NAME, DOMAINS_ASPECT_NAME, SUB_TYPES_ASPECT_NAME, - DATA_PLATFORM_INSTANCE_ASPECT_NAME); + DATA_PLATFORM_INSTANCE_ASPECT_NAME, + BROWSE_PATHS_V2_ASPECT_NAME); private final EntityClient _entityClient; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/mappers/NotebookMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/mappers/NotebookMapper.java index 610bdb309114d..d3b355c13c8f8 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/mappers/NotebookMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/mappers/NotebookMapper.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql.types.notebook.mappers; +import com.linkedin.common.BrowsePathsV2; import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.GlobalTags; import com.linkedin.common.GlossaryTerms; @@ -23,6 +24,7 @@ import com.linkedin.datahub.graphql.generated.QueryCell; import com.linkedin.datahub.graphql.generated.TextCell; import com.linkedin.datahub.graphql.types.common.mappers.AuditStampMapper; +import com.linkedin.datahub.graphql.types.common.mappers.BrowsePathsV2Mapper; import com.linkedin.datahub.graphql.types.common.mappers.ChangeAuditStampsMapper; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; @@ -78,6 +80,8 @@ public Notebook apply(EntityResponse response) { mappingHelper.mapToResult(GLOSSARY_TERMS_ASPECT_NAME, (notebook, dataMap) -> notebook.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap), entityUrn))); mappingHelper.mapToResult(DATA_PLATFORM_INSTANCE_ASPECT_NAME, this::mapDataPlatformInstance); + mappingHelper.mapToResult(BROWSE_PATHS_V2_ASPECT_NAME, (notebook, dataMap) -> + notebook.setBrowsePathV2(BrowsePathsV2Mapper.map(new BrowsePathsV2(dataMap)))); return mappingHelper.getResult(); } diff --git a/datahub-graphql-core/src/main/resources/app.graphql b/datahub-graphql-core/src/main/resources/app.graphql index 9e5016ded3ab4..1d077d4be212b 100644 --- a/datahub-graphql-core/src/main/resources/app.graphql +++ b/datahub-graphql-core/src/main/resources/app.graphql @@ -379,6 +379,16 @@ type FeatureFlagsConfig { Right now this only affects ability to edit user profile image URL but can be extended. """ readOnlyModeEnabled: Boolean! + + """ + Whether search filters V2 should be shown or the default filter side-panel + """ + showSearchFiltersV2: Boolean! + + """ + Whether browse V2 sidebar should be shown + """ + showBrowseV2: Boolean! } """ diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index a45c479cb5143..78f6d8430032d 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -1302,6 +1302,11 @@ type Dataset implements EntityWithRelationships & Entity & BrowsableEntity { """ browsePaths: [BrowsePath!] + """ + The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null. + """ + browsePathV2: BrowsePathV2 + """ Experimental! The resolved health statuses of the Dataset """ @@ -4481,6 +4486,11 @@ type Notebook implements Entity & BrowsableEntity { """ browsePaths: [BrowsePath!] + """ + The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null. + """ + browsePathV2: BrowsePathV2 + """ Whether or not this entity exists on DataHub """ @@ -4748,6 +4758,11 @@ type Dashboard implements EntityWithRelationships & Entity & BrowsableEntity { """ browsePaths: [BrowsePath!] + """ + The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null. + """ + browsePathV2: BrowsePathV2 + """ Experimental (Subject to breaking change) -- Statistics about how this Dashboard is used """ @@ -5063,6 +5078,11 @@ type Chart implements EntityWithRelationships & Entity & BrowsableEntity { """ browsePaths: [BrowsePath!] + """ + The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null. + """ + browsePathV2: BrowsePathV2 + """ Deprecated, use properties field instead Additional read only information about the chart @@ -5425,6 +5445,11 @@ type DataFlow implements EntityWithRelationships & Entity & BrowsableEntity { """ browsePaths: [BrowsePath!] + """ + The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null. + """ + browsePathV2: BrowsePathV2 + """ Deprecated, use properties field instead Additional read only information about a Data flow @@ -5611,6 +5636,11 @@ type DataJob implements EntityWithRelationships & Entity & BrowsableEntity { """ browsePaths: [BrowsePath!] + """ + The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null. + """ + browsePathV2: BrowsePathV2 + """ Deprecated, use properties field instead Additional read only information about a Data processing job @@ -8499,6 +8529,11 @@ type MLModel implements EntityWithRelationships & Entity & BrowsableEntity { """ browsePaths: [BrowsePath!] + """ + The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null. + """ + browsePathV2: BrowsePathV2 + """ The structured glossary terms associated with the entity """ @@ -8600,6 +8635,11 @@ type MLModelGroup implements EntityWithRelationships & Entity & BrowsableEntity """ browsePaths: [BrowsePath!] + """ + The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null. + """ + browsePathV2: BrowsePathV2 + """ Tags applied to entity """ @@ -8704,6 +8744,11 @@ type MLFeature implements EntityWithRelationships & Entity { """ deprecation: Deprecation + """ + The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null. + """ + browsePathV2: BrowsePathV2 + """ The specific instance of the data platform that this entity belongs to """ @@ -9010,6 +9055,11 @@ type MLFeatureTable implements EntityWithRelationships & Entity & BrowsableEntit """ browsePaths: [BrowsePath!] + """ + The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null. + """ + browsePathV2: BrowsePathV2 + """ Tags applied to entity """ diff --git a/datahub-graphql-core/src/main/resources/search.graphql b/datahub-graphql-core/src/main/resources/search.graphql index b897410ea8dfa..67673aa27ca5a 100644 --- a/datahub-graphql-core/src/main/resources/search.graphql +++ b/datahub-graphql-core/src/main/resources/search.graphql @@ -60,6 +60,13 @@ extend type Query { List Data Product assets for a given urn """ listDataProductAssets(urn: String!, input: SearchAcrossEntitiesInput!): SearchResults + + """ + Browse for different entities by getting organizational groups and their + aggregated counts + content. Uses browsePathsV2 aspect and replaces our old + browse endpoint. + """ + browseV2(input: BrowseV2Input!): BrowseResultsV2 } """ @@ -963,6 +970,29 @@ type BrowsePath { path: [String!]! } +""" +A hierarchical entity path V2 +""" +type BrowsePathV2 { + """ + The components of the browse path + """ + path: [BrowsePathEntry!]! +} + +type BrowsePathEntry { + """ + The path name of a group of browse results + """ + name: String! + + """ + An optional entity associated with this browse entry. This will usually be a container entity. + If this entity is not populated, the name must be used. + """ + entity: Entity +} + """ A set of filter criteria """ @@ -1070,3 +1100,99 @@ type SystemFreshness { """ freshnessMillis: Long! } + +""" +Input required for browse queries +""" +input BrowseV2Input { + """ + The browse entity type + """ + type: EntityType! + + """ + The browse path V2 - a list with each entry being part of the browse path V2 + """ + path: [String!] + + """ + The starting point of paginated results + """ + start: Int + + """ + The number of elements included in the results + """ + count: Int + + """ + A list of disjunctive criterion for the filter. (or operation to combine filters) + """ + orFilters: [AndFilterInput!] + + """ + Optional - A View to apply when generating results + """ + viewUrn: String + + """ + The search query string + """ + query: String +} + +""" +The results of a browse path V2 traversal query +""" +type BrowseResultsV2 { + """ + The groups present at the provided browse path V2 + """ + groups: [BrowseResultGroupV2!]! + + """ + The starting point of paginated results + """ + start: Int! + + """ + The number of groups included in the results + """ + count: Int! + + """ + The total number of browse groups under the path with filters applied + """ + total: Int! + + """ + Metadata containing resulting browse groups + """ + metadata: BrowseResultMetadata! +} + +""" +A group of Entities under a given browse path +""" +type BrowseResultGroupV2 { + """ + The path name of a group of browse results + """ + name: String! + + """ + An optional entity associated with this browse group. This will usually be a container entity. + If this entity is not populated, the name must be used. + """ + entity: Entity + + """ + The number of entities within the group + """ + count: Long! + + """ + Whether or not there are any more groups underneath this group + """ + hasSubGroups: Boolean! +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseV2ResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseV2ResolverTest.java new file mode 100644 index 0000000000000..4a948537ab4fe --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseV2ResolverTest.java @@ -0,0 +1,271 @@ +package com.linkedin.datahub.graphql.resolvers.browse; + +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.StringArray; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.AndFilterInput; +import com.linkedin.datahub.graphql.generated.BrowseResultsV2; +import com.linkedin.datahub.graphql.generated.BrowseV2Input; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.FacetFilterInput; +import com.linkedin.datahub.graphql.resolvers.ResolverUtils; +import com.linkedin.datahub.graphql.resolvers.chart.BrowseV2Resolver; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.browse.BrowseResultGroupV2; +import com.linkedin.metadata.browse.BrowseResultGroupV2Array; +import com.linkedin.metadata.browse.BrowseResultMetadata; +import com.linkedin.metadata.browse.BrowseResultV2; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.service.ViewService; +import com.linkedin.view.DataHubViewDefinition; +import com.linkedin.view.DataHubViewInfo; +import com.linkedin.view.DataHubViewType; +import graphql.schema.DataFetchingEnvironment; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.List; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; + +public class BrowseV2ResolverTest { + + private static final Urn TEST_USER_URN = UrnUtils.getUrn("urn:li:corpuser:test"); + private static final Urn TEST_VIEW_URN = UrnUtils.getUrn("urn:li:dataHubView:test"); + + @Test + public static void testBrowseV2Success() throws Exception { + ViewService mockService = Mockito.mock(ViewService.class); + EntityClient mockClient = initMockEntityClient( + "dataset", + "␟test␟path", + "*", + null, + 0, + 10, + new BrowseResultV2() + .setNumGroups(2) + .setGroups(new BrowseResultGroupV2Array( + new BrowseResultGroupV2().setCount(5).setName("first group").setHasSubGroups(true), + new BrowseResultGroupV2().setCount(4).setName("second group").setHasSubGroups(false) + )) + .setMetadata(new BrowseResultMetadata().setPath("␟test␟path").setTotalNumEntities(100)) + .setFrom(0) + .setPageSize(10) + ); + + final BrowseV2Resolver resolver = new BrowseV2Resolver(mockClient, mockService); + + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + BrowseV2Input input = new BrowseV2Input(); + input.setPath(ImmutableList.of("test", "path")); + input.setType(EntityType.DATASET); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + BrowseResultsV2 result = resolver.get(mockEnv).get(); + + compareResultToExpectedData(result, getExpectedResult()); + } + + @Test + public static void testBrowseV2SuccessWithQueryAndFilter() throws Exception { + ViewService mockService = Mockito.mock(ViewService.class); + + List orFilters = new ArrayList<>(); + AndFilterInput andFilterInput = new AndFilterInput(); + FacetFilterInput facetFilterInput = new FacetFilterInput(); + facetFilterInput.setField("owners"); + facetFilterInput.setValues(ImmutableList.of("urn:li:corpuser:test")); + andFilterInput.setAnd(ImmutableList.of(facetFilterInput)); + orFilters.add(andFilterInput); + Filter filter = ResolverUtils.buildFilter(null, orFilters); + + EntityClient mockClient = initMockEntityClient( + "dataset", + "␟test␟path", + "test", + filter, + 0, + 10, + new BrowseResultV2() + .setNumGroups(2) + .setGroups(new BrowseResultGroupV2Array( + new BrowseResultGroupV2().setCount(5).setName("first group").setHasSubGroups(true), + new BrowseResultGroupV2().setCount(4).setName("second group").setHasSubGroups(false) + )) + .setMetadata(new BrowseResultMetadata().setPath("␟test␟path").setTotalNumEntities(100)) + .setFrom(0) + .setPageSize(10) + ); + + final BrowseV2Resolver resolver = new BrowseV2Resolver(mockClient, mockService); + + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + BrowseV2Input input = new BrowseV2Input(); + input.setPath(ImmutableList.of("test", "path")); + input.setType(EntityType.DATASET); + input.setQuery("test"); + input.setOrFilters(orFilters); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + BrowseResultsV2 result = resolver.get(mockEnv).get(); + + compareResultToExpectedData(result, getExpectedResult()); + } + + @Test + public static void testBrowseV2SuccessWithView() throws Exception { + DataHubViewInfo viewInfo = createViewInfo(new StringArray()); + ViewService viewService = initMockViewService(TEST_VIEW_URN, viewInfo); + + EntityClient mockClient = initMockEntityClient( + "dataset", + "␟test␟path", + "*", + viewInfo.getDefinition().getFilter(), + 0, + 10, + new BrowseResultV2() + .setNumGroups(2) + .setGroups(new BrowseResultGroupV2Array( + new BrowseResultGroupV2().setCount(5).setName("first group").setHasSubGroups(true), + new BrowseResultGroupV2().setCount(4).setName("second group").setHasSubGroups(false) + )) + .setMetadata(new BrowseResultMetadata().setPath("␟test␟path").setTotalNumEntities(100)) + .setFrom(0) + .setPageSize(10) + ); + + final BrowseV2Resolver resolver = new BrowseV2Resolver(mockClient, viewService); + + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + BrowseV2Input input = new BrowseV2Input(); + input.setPath(ImmutableList.of("test", "path")); + input.setType(EntityType.DATASET); + input.setViewUrn(TEST_VIEW_URN.toString()); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + BrowseResultsV2 result = resolver.get(mockEnv).get(); + + compareResultToExpectedData(result, getExpectedResult()); + } + + private static void compareResultToExpectedData(BrowseResultsV2 result, BrowseResultsV2 expected) { + Assert.assertEquals(result.getCount(), expected.getCount()); + Assert.assertEquals(result.getStart(), expected.getStart()); + Assert.assertEquals(result.getTotal(), expected.getTotal()); + Assert.assertEquals(result.getGroups().size(), expected.getGroups().size()); + result.getGroups().forEach(group -> { + Assert.assertTrue(expected.getGroups().stream().filter(g -> g.getName().equals(group.getName())).count() > 0); + }); + Assert.assertEquals(result.getMetadata().getPath(), expected.getMetadata().getPath()); + Assert.assertEquals(result.getMetadata().getTotalNumEntities(), expected.getMetadata().getTotalNumEntities()); + } + + private static BrowseResultsV2 getExpectedResult() { + BrowseResultsV2 results = new BrowseResultsV2(); + results.setTotal(2); + results.setStart(0); + results.setCount(10); + + List groups = new ArrayList<>(); + com.linkedin.datahub.graphql.generated.BrowseResultGroupV2 browseGroup1 = new com.linkedin.datahub.graphql.generated.BrowseResultGroupV2(); + browseGroup1.setName("first group"); + browseGroup1.setCount(5L); + browseGroup1.setHasSubGroups(true); + groups.add(browseGroup1); + com.linkedin.datahub.graphql.generated.BrowseResultGroupV2 browseGroup2 = new com.linkedin.datahub.graphql.generated.BrowseResultGroupV2(); + browseGroup2.setName("second group"); + browseGroup2.setCount(4L); + browseGroup2.setHasSubGroups(false); + groups.add(browseGroup2); + results.setGroups(groups); + + com.linkedin.datahub.graphql.generated.BrowseResultMetadata resultMetadata = new com.linkedin.datahub.graphql.generated.BrowseResultMetadata(); + resultMetadata.setPath(ImmutableList.of("test", "path")); + resultMetadata.setTotalNumEntities(100L); + results.setMetadata(resultMetadata); + + return results; + } + + private static EntityClient initMockEntityClient( + String entityName, + String path, + String query, + Filter filter, + int start, + int limit, + BrowseResultV2 result + ) throws Exception { + EntityClient client = Mockito.mock(EntityClient.class); + Mockito.when(client.browseV2( + Mockito.eq(entityName), + Mockito.eq(path), + Mockito.eq(filter), + Mockito.eq(query), + Mockito.eq(start), + Mockito.eq(limit), + Mockito.any(Authentication.class) + )).thenReturn( + result + ); + return client; + } + + private static ViewService initMockViewService( + Urn viewUrn, + DataHubViewInfo viewInfo + ) { + ViewService service = Mockito.mock(ViewService.class); + Mockito.when(service.getViewInfo( + Mockito.eq(viewUrn), + Mockito.any(Authentication.class) + )).thenReturn( + viewInfo + ); + return service; + } + private static DataHubViewInfo createViewInfo(StringArray entityNames) { + Filter viewFilter = new Filter() + .setOr(new ConjunctiveCriterionArray( + new ConjunctiveCriterion().setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("field") + .setValue("test") + .setValues(new StringArray(ImmutableList.of("test"))) + )) + ))); + + DataHubViewInfo info = new DataHubViewInfo(); + info.setName("test"); + info.setType(DataHubViewType.GLOBAL); + info.setCreated(new AuditStamp().setTime(0L).setActor(TEST_USER_URN)); + info.setLastModified(new AuditStamp().setTime(0L).setActor(TEST_USER_URN)); + info.setDefinition(new DataHubViewDefinition() + .setEntityTypes(entityNames) + .setFilter(viewFilter) + ); + return info; + } + + private BrowseV2ResolverTest() { } + +} diff --git a/datahub-web-react/src/Mocks.tsx b/datahub-web-react/src/Mocks.tsx index 441b090c974d1..8302fb09af4af 100644 --- a/datahub-web-react/src/Mocks.tsx +++ b/datahub-web-react/src/Mocks.tsx @@ -28,17 +28,19 @@ import { Container, PlatformPrivileges, FilterOperator, + AppConfig, } from './types.generated'; import { GetTagDocument } from './graphql/tag.generated'; import { GetMlModelDocument } from './graphql/mlModel.generated'; import { GetMlModelGroupDocument } from './graphql/mlModelGroup.generated'; import { GetGlossaryTermDocument, GetGlossaryTermQuery } from './graphql/glossaryTerm.generated'; -import { GetEntityCountsDocument } from './graphql/app.generated'; +import { GetEntityCountsDocument, AppConfigDocument } from './graphql/app.generated'; import { GetMeDocument } from './graphql/me.generated'; import { ListRecommendationsDocument } from './graphql/recommendations.generated'; import { FetchedEntity } from './app/lineage/types'; +import { DEFAULT_APP_CONFIG } from './appConfigContext'; -const user1 = { +export const user1 = { username: 'sdas', urn: 'urn:li:corpuser:1', type: EntityType.CorpUser, @@ -119,7 +121,7 @@ const user2 = { settings: { appearance: { showSimplifiedHomepage: false }, views: { defaultView: null } }, }; -const dataPlatform = { +export const dataPlatform = { urn: 'urn:li:dataPlatform:hdfs', name: 'HDFS', type: EntityType.DataPlatform, @@ -127,7 +129,22 @@ const dataPlatform = { displayName: 'HDFS', type: PlatformType.FileSystem, datasetNameDelimiter: '.', - logoUrl: '', + logoUrl: + 'https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/lookerlogo.png', + }, +}; + +export const dataPlatformInstance = { + urn: 'urn:li:dataPlatformInstance:(urn:li:dataPlatform:clickhouse,clickhousetestserver)', + type: EntityType.DataPlatformInstance, + instanceId: 'clickhousetestserver', + platform: { + type: 'DATA_PLATFORM', + urn: 'urn:li:dataPlatform:clickhouse', + properties: { + displayName: 'ClickHouse', + logoUrl: '/assets/platforms/clickhouselogo.png', + }, }, }; @@ -238,6 +255,7 @@ export const dataset1 = { testResults: null, statsSummary: null, embed: null, + browsePathV2: { path: [{ name: 'test', entity: null }], __typename: 'BrowsePathV2' }, }; export const dataset2 = { @@ -331,6 +349,7 @@ export const dataset2 = { testResults: null, statsSummary: null, embed: null, + browsePathV2: { path: [{ name: 'test', entity: null }], __typename: 'BrowsePathV2' }, }; export const dataset3 = { @@ -508,6 +527,7 @@ export const dataset3 = { siblings: null, statsSummary: null, embed: null, + browsePathV2: { path: [{ name: 'test', entity: null }], __typename: 'BrowsePathV2' }, } as Dataset; export const dataset3WithSchema = { @@ -3548,6 +3568,26 @@ export const mocks = [ }, ]; +export const mocksWithSearchFlagsOff = [ + ...mocks, + { + request: { + query: AppConfigDocument, + }, + result: { + data: { + appConfig: { + ...DEFAULT_APP_CONFIG, + featureFlags: { + ...DEFAULT_APP_CONFIG.featureFlags, + showSearchFiltersV2: false, + }, + } as AppConfig, + }, + }, + }, +]; + export const platformPrivileges: PlatformPrivileges = { viewAnalytics: true, managePolicies: true, diff --git a/datahub-web-react/src/app/analytics/event.ts b/datahub-web-react/src/app/analytics/event.ts index fa1c9e938f860..9cb14f5aa127b 100644 --- a/datahub-web-react/src/app/analytics/event.ts +++ b/datahub-web-react/src/app/analytics/event.ts @@ -1,6 +1,7 @@ import { DataHubViewType, EntityType, RecommendationRenderType, ScenarioType } from '../../types.generated'; import { EmbedLookupNotFoundReason } from '../embed/lookup/constants'; import { Direction } from '../lineage/types'; +import { FilterMode } from '../search/utils/constants'; /** * Valid event types. @@ -15,8 +16,13 @@ export enum EventType { SearchResultsViewEvent, SearchResultClickEvent, EntitySearchResultClickEvent, + SearchFiltersClearAllEvent, + SearchFiltersShowMoreEvent, BrowseResultClickEvent, HomePageBrowseResultClickEvent, + BrowseV2ToggleSidebarEvent, + BrowseV2ToggleNodeEvent, + BrowseV2SelectNodeEvent, EntityViewEvent, EntitySectionViewEvent, EntityActionEvent, @@ -161,6 +167,11 @@ export interface SearchResultsViewEvent extends BaseEvent { entityTypeFilter?: EntityType; page?: number; total: number; + entityTypes: string[]; + filterFields: string[]; + filterCount: number; + filterMode: FilterMode; + searchVersion: string; } /** @@ -176,6 +187,17 @@ export interface SearchResultClickEvent extends BaseEvent { total: number; } +export interface SearchFiltersClearAllEvent extends BaseEvent { + type: EventType.SearchFiltersClearAllEvent; + total: number; +} + +export interface SearchFiltersShowMoreEvent extends BaseEvent { + type: EventType.SearchFiltersShowMoreEvent; + activeFilterCount: number; + hiddenFilterCount: number; +} + /** * Logged on user browse result click. */ @@ -196,6 +218,40 @@ export interface HomePageBrowseResultClickEvent extends BaseEvent { entityType: EntityType; } +/** + * Logged when a user opens or closes the browse v2 sidebar + */ +export interface BrowseV2ToggleSidebarEvent extends BaseEvent { + type: EventType.BrowseV2ToggleSidebarEvent; + action: 'open' | 'close'; +} + +/** + * Logged when a user opens or closes a sidebar node + */ +export interface BrowseV2ToggleNodeEvent extends BaseEvent { + type: EventType.BrowseV2ToggleNodeEvent; + targetNode: 'entity' | 'environment' | 'platform' | 'browse'; + action: 'open' | 'close'; + entity: string; + environment?: string; + platform?: string; + targetDepth: number; +} + +/** + * Logged when a user selects a browse node in the sidebar + */ +export interface BrowseV2SelectNodeEvent extends BaseEvent { + type: EventType.BrowseV2SelectNodeEvent; + targetNode: 'browse'; + action: 'select' | 'deselect'; + entity: string; + environment?: string; + platform?: string; + targetDepth: number; +} + /** * Logged when user views an entity profile. */ @@ -437,7 +493,10 @@ export interface ManuallyDeleteLineageEvent extends BaseEvent { */ export interface CreateViewEvent extends BaseEvent { type: EventType.CreateViewEvent; - viewType: DataHubViewType; + viewType?: DataHubViewType; + filterFields: string[]; + entityTypes: string[]; + searchVersion: string; } /** @@ -445,8 +504,11 @@ export interface CreateViewEvent extends BaseEvent { */ export interface UpdateViewEvent extends BaseEvent { type: EventType.UpdateViewEvent; - viewType: DataHubViewType; + viewType?: DataHubViewType; urn: string; + filterFields: string[]; + entityTypes: string[]; + searchVersion: string; } /** @@ -544,8 +606,13 @@ export type Event = | SearchResultsExploreAllClickEvent | SearchResultsViewEvent | SearchResultClickEvent + | SearchFiltersClearAllEvent + | SearchFiltersShowMoreEvent | BrowseResultClickEvent | HomePageBrowseResultClickEvent + | BrowseV2ToggleSidebarEvent + | BrowseV2ToggleNodeEvent + | BrowseV2SelectNodeEvent | EntityViewEvent | EntitySectionViewEvent | EntityActionEvent diff --git a/datahub-web-react/src/app/entity/EntityRegistry.tsx b/datahub-web-react/src/app/entity/EntityRegistry.tsx index ca76640ac85d8..a07fd02841197 100644 --- a/datahub-web-react/src/app/entity/EntityRegistry.tsx +++ b/datahub-web-react/src/app/entity/EntityRegistry.tsx @@ -35,6 +35,10 @@ export default class EntityRegistry { return validatedGet(type, this.entityTypeToEntity); } + hasEntity(type: EntityType): boolean { + return this.entityTypeToEntity.has(type); + } + getEntities(): Array> { return this.entities; } diff --git a/datahub-web-react/src/app/entity/shared/components/styled/StyledTag.tsx b/datahub-web-react/src/app/entity/shared/components/styled/StyledTag.tsx index 22b9819001a5f..c1a23811fdd7e 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/StyledTag.tsx +++ b/datahub-web-react/src/app/entity/shared/components/styled/StyledTag.tsx @@ -2,7 +2,7 @@ import { Tag } from 'antd'; import styled, { css } from 'styled-components'; import ColorHash from 'color-hash'; -const generateColor = new ColorHash({ +export const generateColor = new ColorHash({ saturation: 0.9, }); diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/nav/EntityProfileNavBar.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/nav/EntityProfileNavBar.tsx index d0941f15581e3..1b324412fbe09 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/nav/EntityProfileNavBar.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/nav/EntityProfileNavBar.tsx @@ -5,6 +5,8 @@ import { useGetBrowsePathsQuery } from '../../../../../../graphql/browse.generat import { EntityType } from '../../../../../../types.generated'; import { useEntityRegistry } from '../../../../../useEntityRegistry'; import { ProfileNavBrowsePath } from './ProfileNavBrowsePath'; +import ProfileNavBrowsePathV2 from './ProfileNavBrowsePathV2'; +import { useIsBrowseV2 } from '../../../../../search/useSearchAndBrowseVersion'; type Props = { urn: string; @@ -14,6 +16,7 @@ type Props = { const AffixWithHeight = styled(Affix)``; export const EntityProfileNavBar = ({ urn, entityType }: Props) => { + const showBrowseV2 = useIsBrowseV2(); const { data: browseData } = useGetBrowsePathsQuery({ variables: { input: { urn, type: entityType } }, fetchPolicy: 'cache-first', @@ -23,12 +26,15 @@ export const EntityProfileNavBar = ({ urn, entityType }: Props) => { return ( - + {showBrowseV2 && } + {!showBrowseV2 && ( + + )} ); }; diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/nav/ProfileNavBrowsePath.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/nav/ProfileNavBrowsePath.tsx index 1c4ff82052b19..1026251ac629b 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/nav/ProfileNavBrowsePath.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/nav/ProfileNavBrowsePath.tsx @@ -8,7 +8,7 @@ import { PageRoutes } from '../../../../../../conf/Global'; import { ANTD_GRAY } from '../../../constants'; import { LineageSelector } from './LineageSelector'; -const BrowseRow = styled(Row)` +export const BrowseRow = styled(Row)` padding: 10px 20px; background-color: ${(props) => props.theme.styles['body-background']}; display: flex; @@ -19,6 +19,7 @@ const BrowseRow = styled(Row)` export const BreadcrumbItem = styled(Breadcrumb.Item)<{ disabled?: boolean }>` &&& :hover { color: ${(props) => (props.disabled ? ANTD_GRAY[7] : props.theme.styles['primary-color'])}; + cursor: pointer; } `; @@ -50,6 +51,7 @@ export const ProfileNavBrowsePath = ({ urn, type, path, breadcrumbLinksEnabled } ? '#' : `${baseBrowsePath}/${createPartialPath(path.slice(0, index + 1))}` } + data-testid={`legacy-browse-path-${part}`} > {part} diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/nav/ProfileNavBrowsePathV2.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/nav/ProfileNavBrowsePathV2.tsx new file mode 100644 index 0000000000000..d91bb565c0cd1 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/containers/profile/nav/ProfileNavBrowsePathV2.tsx @@ -0,0 +1,114 @@ +import React from 'react'; +import { useHistory } from 'react-router'; +import { Breadcrumb } from 'antd'; +import styled from 'styled-components'; +import { BreadcrumbItem, BrowseRow } from './ProfileNavBrowsePath'; +import { useEntityData } from '../../../EntityContext'; +import { useEntityRegistry } from '../../../../../useEntityRegistry'; +import { navigateToSearchUrl } from '../../../../../search/utils/navigateToSearchUrl'; +import { BrowsePathV2, EntityType, FabricType, FacetFilterInput } from '../../../../../../types.generated'; +import { + BROWSE_PATH_V2_FILTER_NAME, + ENTITY_SUB_TYPE_FILTER_NAME, + ORIGIN_FILTER_NAME, + PLATFORM_FILTER_NAME, +} from '../../../../../search/utils/constants'; +import useHasMultipleEnvironmentsQuery from './useHasMultipleEnvironmentsQuery'; +import { createBrowseV2SearchFilter } from '../../../../../search/filters/utils'; +import { LineageSelector } from './LineageSelector'; + +const StyledBreadcrumb = styled(Breadcrumb)` + font-size: 16px; +`; + +interface Props { + urn: string; + type: EntityType; +} + +export default function ProfileNavBrowsePathV2({ urn, type }: Props) { + const history = useHistory(); + const entityRegistry = useEntityRegistry(); + const { entityData } = useEntityData(); + const isBrowsable = entityRegistry.getBrowseEntityTypes().includes(type); + const hasEnvironment = !!entityData?.origin; + + const hasMultipleEnvironments = useHasMultipleEnvironmentsQuery(type); + + function handlePathClick(filters: FacetFilterInput[]) { + navigateToSearchUrl({ query: '*', filters, history }); + } + + function generateFiltersForPlatform() { + const filters: FacetFilterInput[] = [{ field: ENTITY_SUB_TYPE_FILTER_NAME, values: [type] }]; + if (hasMultipleEnvironments && hasEnvironment) { + filters.push({ field: ORIGIN_FILTER_NAME, values: [entityData?.origin as FabricType] }); + } + if (entityData?.platform) { + filters.push({ field: PLATFORM_FILTER_NAME, values: [entityData.platform.urn] }); + } + return filters; + } + + function generateFiltersForBrowsePath(path: string[]) { + const filters = generateFiltersForPlatform(); + const pathValue = createBrowseV2SearchFilter(path); + filters.push({ field: BROWSE_PATH_V2_FILTER_NAME, values: [pathValue] }); + return filters; + } + + return ( + + + handlePathClick([{ field: ENTITY_SUB_TYPE_FILTER_NAME, values: [type] }])} + > + {entityRegistry.getCollectionName(type)} + + {hasMultipleEnvironments && hasEnvironment && ( + + handlePathClick([ + { field: ENTITY_SUB_TYPE_FILTER_NAME, values: [type] }, + { field: ORIGIN_FILTER_NAME, values: [entityData?.origin as FabricType] }, + ]) + } + > + {entityData?.origin} + + )} + {entityData?.platform && ( + handlePathClick(generateFiltersForPlatform())} + > + {entityRegistry.getDisplayName(EntityType.DataPlatform, entityData.platform)} + + )} + {entityData?.browsePathV2?.path.map((pathEntry, index) => ( + + handlePathClick( + generateFiltersForBrowsePath([ + ...(entityData.browsePathV2 as BrowsePathV2).path + .slice(0, index + 1) + .map((e) => e?.name || ''), + ]), + ) + } + data-testid={`browse-path-${pathEntry?.name}`} + > + {pathEntry.entity + ? entityRegistry.getDisplayName(pathEntry.entity.type, pathEntry.entity) + : pathEntry.name} + + ))} + + + + ); +} diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/nav/useHasMultipleEnvironmentsQuery.ts b/datahub-web-react/src/app/entity/shared/containers/profile/nav/useHasMultipleEnvironmentsQuery.ts new file mode 100644 index 0000000000000..9bcb4ef6e753d --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/containers/profile/nav/useHasMultipleEnvironmentsQuery.ts @@ -0,0 +1,17 @@ +import { useAggregateAcrossEntitiesQuery } from '../../../../../../graphql/search.generated'; +import { ENTITY_FILTER_NAME, ORIGIN_FILTER_NAME } from '../../../../../search/utils/constants'; +import { EntityType } from '../../../../../../types.generated'; + +export default function useHasMultipleEnvironmentsQuery(entityType: EntityType) { + const { data } = useAggregateAcrossEntitiesQuery({ + variables: { + input: { + facets: [ORIGIN_FILTER_NAME], + query: '*', + orFilters: [{ and: [{ field: ENTITY_FILTER_NAME, values: [entityType] }] }], + }, + }, + }); + const environmentAggs = data?.aggregateAcrossEntities?.facets?.find((facet) => facet.field === ORIGIN_FILTER_NAME); + return environmentAggs && environmentAggs.aggregations.length > 1; +} diff --git a/datahub-web-react/src/app/entity/shared/types.ts b/datahub-web-react/src/app/entity/shared/types.ts index 0f160fd957cae..d1fbfe41fc66d 100644 --- a/datahub-web-react/src/app/entity/shared/types.ts +++ b/datahub-web-react/src/app/entity/shared/types.ts @@ -34,6 +34,8 @@ import { FineGrainedLineage, EntityPrivileges, Embed, + FabricType, + BrowsePathV2, } from '../../../types.generated'; import { FetchedEntity } from '../../lineage/types'; @@ -105,6 +107,8 @@ export type GenericEntityProperties = { privileges?: Maybe; embed?: Maybe; exists?: boolean; + origin?: Maybe; + browsePathV2?: Maybe; }; export type GenericEntityUpdate = { diff --git a/datahub-web-react/src/app/entity/view/builder/ViewBuilder.tsx b/datahub-web-react/src/app/entity/view/builder/ViewBuilder.tsx index ac1107d735e8b..8a0b84ce0fa7f 100644 --- a/datahub-web-react/src/app/entity/view/builder/ViewBuilder.tsx +++ b/datahub-web-react/src/app/entity/view/builder/ViewBuilder.tsx @@ -8,8 +8,9 @@ import { updateViewSelectCache, updateListMyViewsCache } from '../cacheUtils'; import { convertStateToUpdateInput, DEFAULT_LIST_VIEWS_PAGE_SIZE } from '../utils'; import { useUserContext } from '../../../context/useUserContext'; import { ViewBuilderMode } from './types'; -import analytics, { Event, EventType } from '../../../analytics'; +import analytics, { EventType } from '../../../analytics'; import { DataHubView } from '../../../../types.generated'; +import { useSearchVersion } from '../../../search/useSearchAndBrowseVersion'; type Props = { mode: ViewBuilderMode; @@ -23,6 +24,7 @@ type Props = { * This component handles creating and editing DataHub Views. */ export const ViewBuilder = ({ mode, urn, initialState, onSubmit, onCancel }: Props) => { + const searchVersion = useSearchVersion(); const userContext = useUserContext(); const client = useApolloClient(); @@ -30,11 +32,17 @@ export const ViewBuilder = ({ mode, urn, initialState, onSubmit, onCancel }: Pro const [createViewMutation] = useCreateViewMutation(); const emitTrackingEvent = (viewUrn: string, state: ViewBuilderState, isCreate: boolean) => { + const filterFields = Array.from(new Set(state.definition?.filter?.filters.map((filter) => filter.field) ?? [])); + const entityTypes = Array.from(new Set(state.definition?.entityTypes ?? [])); + analytics.event({ type: isCreate ? EventType.CreateViewEvent : EventType.UpdateViewEvent, urn: viewUrn, viewType: state.viewType, - } as Event); + filterFields, + entityTypes, + searchVersion, + }); }; /** diff --git a/datahub-web-react/src/app/entity/view/builder/ViewBuilderForm.tsx b/datahub-web-react/src/app/entity/view/builder/ViewBuilderForm.tsx index 2d24f9efd1f17..f1dc8c13489ae 100644 --- a/datahub-web-react/src/app/entity/view/builder/ViewBuilderForm.tsx +++ b/datahub-web-react/src/app/entity/view/builder/ViewBuilderForm.tsx @@ -53,7 +53,7 @@ export const ViewBuilderForm = ({ urn, mode, state, updateState }: Props) => { const isEditing = urn !== undefined; return ( - <> +
Name}> Give your new View a name. @@ -112,6 +112,6 @@ export const ViewBuilderForm = ({ urn, mode, state, updateState }: Props) => {
- + ); }; diff --git a/datahub-web-react/src/app/entity/view/builder/ViewBuilderModal.tsx b/datahub-web-react/src/app/entity/view/builder/ViewBuilderModal.tsx index 4a5e563fa81bd..8491fdfea2675 100644 --- a/datahub-web-react/src/app/entity/view/builder/ViewBuilderModal.tsx +++ b/datahub-web-react/src/app/entity/view/builder/ViewBuilderModal.tsx @@ -79,6 +79,7 @@ export const ViewBuilderModal = ({ mode, urn, initialState, onSubmit, onCancel } visible width={modalWidth} onCancel={onCancel} + data-testid="view-modal" > diff --git a/datahub-web-react/src/app/entity/view/builder/__tests__/utils.test.ts b/datahub-web-react/src/app/entity/view/builder/__tests__/utils.test.ts new file mode 100644 index 0000000000000..ee1c17a8ff49a --- /dev/null +++ b/datahub-web-react/src/app/entity/view/builder/__tests__/utils.test.ts @@ -0,0 +1,23 @@ +import { + ENTITY_FILTER_NAME, + ENTITY_SUB_TYPE_FILTER_NAME, + TYPE_NAMES_FILTER_NAME, +} from '../../../../search/utils/constants'; +import { convertNestedSubTypeFilter } from '../utils'; + +describe('view builder utils', () => { + it('should convert the nested subtypes filter properly along with other filters', () => { + const filters = [ + { field: 'platform', values: ['platform1', 'platform2'] }, + { field: ENTITY_SUB_TYPE_FILTER_NAME, values: ['DATASETS', 'CONTAINERS␞schema'] }, + { field: 'tag', values: ['tag1', 'tag2'] }, + ]; + + expect(convertNestedSubTypeFilter(filters)).toMatchObject([ + { field: 'platform', values: ['platform1', 'platform2'] }, + { field: 'tag', values: ['tag1', 'tag2'] }, + { field: ENTITY_FILTER_NAME, values: ['DATASETS'] }, + { field: TYPE_NAMES_FILTER_NAME, values: ['schema'] }, + ]); + }); +}); diff --git a/datahub-web-react/src/app/entity/view/builder/utils.ts b/datahub-web-react/src/app/entity/view/builder/utils.ts index 3b67face7f33e..6f84dcef2d977 100644 --- a/datahub-web-react/src/app/entity/view/builder/utils.ts +++ b/datahub-web-react/src/app/entity/view/builder/utils.ts @@ -6,7 +6,13 @@ import { FacetFilterInput, LogicalOperator, } from '../../../../types.generated'; -import { ENTITY_FILTER_NAME, UnionType } from '../../../search/utils/constants'; +import { + ENTITY_FILTER_NAME, + ENTITY_SUB_TYPE_FILTER_NAME, + UnionType, + FILTER_DELIMITER, + TYPE_NAMES_FILTER_NAME, +} from '../../../search/utils/constants'; /** * Extract the special "Entity Type" filter values from a list @@ -19,6 +25,37 @@ export const extractEntityTypesFilterValues = (filters: Array) .flatMap((filter) => filter.values as EntityType[]); }; +/** + * Converts the nested subtype filter to be split into entity type filters and subType filters. + * Right now we don't allow mixing of entity type and subType filters when creating a view since + * this filter requires an OR between entity type and subType but AND between other filter types (like tags). + * Example: { field: "_entityType␞typeNames" values: ["DATASETS␞table"] } -> { field: "typeNames", values: ["table"]} + * Example: { field: "_entityType␞typeNames" values: ["DATASETS", "CONTAINERS"] } -> { field: "_entityType", values: ["DATASETS", "CONTAINERS"]} + */ +export function convertNestedSubTypeFilter(filters: Array) { + const convertedFilters = filters.filter((f) => f.field !== ENTITY_SUB_TYPE_FILTER_NAME) || []; + const nestedSubTypeFilter = filters.find((f) => f.field === ENTITY_SUB_TYPE_FILTER_NAME); + if (nestedSubTypeFilter) { + const entityTypeFilterValues: string[] = []; + const subTypeFilterValues: string[] = []; + nestedSubTypeFilter.values?.forEach((value) => { + if (!value.includes(FILTER_DELIMITER)) { + entityTypeFilterValues.push(value); + } else { + const nestedValues = value.split(FILTER_DELIMITER); + subTypeFilterValues.push(nestedValues[nestedValues.length - 1]); + } + }); + if (entityTypeFilterValues.length) { + convertedFilters.push({ field: ENTITY_FILTER_NAME, values: entityTypeFilterValues }); + } + if (subTypeFilterValues.length) { + convertedFilters.push({ field: TYPE_NAMES_FILTER_NAME, values: subTypeFilterValues }); + } + } + return convertedFilters; +} + /** * Build an object representation of a View Definition, which consists of a list of entity types + * a set of filters joined in either conjunction or disjunction. @@ -27,8 +64,9 @@ export const extractEntityTypesFilterValues = (filters: Array) * @param operatorType a logical operator to be used when joining the filters into the View definition. */ export const buildViewDefinition = (filters: Array, operatorType: LogicalOperator) => { - const entityTypes = extractEntityTypesFilterValues(filters); - const filteredFilters = filters.filter((filter) => filter.field !== ENTITY_FILTER_NAME); + const convertedFilters = convertNestedSubTypeFilter(filters); + const entityTypes = extractEntityTypesFilterValues(convertedFilters); + const filteredFilters = convertedFilters.filter((filter) => filter.field !== ENTITY_FILTER_NAME); return { entityTypes, filter: { diff --git a/datahub-web-react/src/app/onboarding/OnboardingStep.tsx b/datahub-web-react/src/app/onboarding/OnboardingStep.tsx index 9c2d83351afbe..08bad906724f0 100644 --- a/datahub-web-react/src/app/onboarding/OnboardingStep.tsx +++ b/datahub-web-react/src/app/onboarding/OnboardingStep.tsx @@ -1,7 +1,11 @@ import { ReactNode } from 'react'; +// Conditional steps are steps with a prerequisiteStepId. We should only it show if the user has +// already seen its pre-requisite step. If they have not, then they should never see the conditional +// step since they will see the pre-requisite step next time they encounter it on the screen. export type OnboardingStep = { id?: string; + prerequisiteStepId?: string; title?: string | ReactNode; content?: ReactNode; selector?: string; diff --git a/datahub-web-react/src/app/onboarding/OnboardingTour.tsx b/datahub-web-react/src/app/onboarding/OnboardingTour.tsx index ebdcb8453f3a6..e738cbab6b6be 100644 --- a/datahub-web-react/src/app/onboarding/OnboardingTour.tsx +++ b/datahub-web-react/src/app/onboarding/OnboardingTour.tsx @@ -5,7 +5,7 @@ import { useBatchUpdateStepStatesMutation } from '../../graphql/step.generated'; import { EducationStepsContext } from '../../providers/EducationStepsContext'; import { StepStateResult } from '../../types.generated'; import { useUserContext } from '../context/useUserContext'; -import { convertStepId, getStepsToRender } from './utils'; +import { convertStepId, getConditionalStepIdsToAdd, getStepsToRender } from './utils'; type Props = { stepIds: string[]; @@ -42,7 +42,10 @@ export const OnboardingTour = ({ stepIds }: Props) => { function closeTour() { setIsOpen(false); setReshow(false); - const convertedIds = filteredStepIds.map((id) => convertStepId(id, userUrn || '')); + // add conditional steps where its pre-requisite step ID is in our list of IDs we mark as completed + const conditionalStepIds = getConditionalStepIdsToAdd(stepIds, filteredStepIds); + const finalStepIds = [...filteredStepIds, ...conditionalStepIds]; + const convertedIds = finalStepIds.map((id) => convertStepId(id, userUrn || '')); const stepStates = convertedIds.map((id) => ({ id, properties: [] })); batchUpdateStepStates({ variables: { input: { states: stepStates } } }).then(() => { const results = convertedIds.map((id) => ({ id, properties: [{}] } as StepStateResult)); diff --git a/datahub-web-react/src/app/onboarding/config/SearchOnboardingConfig.tsx b/datahub-web-react/src/app/onboarding/config/SearchOnboardingConfig.tsx index ee150bef0c3e4..bec7a6e4cc226 100644 --- a/datahub-web-react/src/app/onboarding/config/SearchOnboardingConfig.tsx +++ b/datahub-web-react/src/app/onboarding/config/SearchOnboardingConfig.tsx @@ -4,12 +4,14 @@ import { OnboardingStep } from '../OnboardingStep'; export const SEARCH_RESULTS_FILTERS_ID = 'search-results-filters'; export const SEARCH_RESULTS_ADVANCED_SEARCH_ID = 'search-results-advanced-search'; +export const SEARCH_RESULTS_BROWSE_SIDEBAR_ID = 'search-results-browse-sidebar'; +export const SEARCH_RESULTS_FILTERS_V2_INTRO = 'search-results-filters-v2-intro'; export const SearchOnboardingConfig: OnboardingStep[] = [ { id: SEARCH_RESULTS_FILTERS_ID, selector: `#${SEARCH_RESULTS_FILTERS_ID}`, - title: 'Narrow your search ⚡', + title: '🕵️ Narrow your search', content: ( Quickly find relevant assets by applying one or more filters. Try filtering by Type,{' '} @@ -20,10 +22,35 @@ export const SearchOnboardingConfig: OnboardingStep[] = [ { id: SEARCH_RESULTS_ADVANCED_SEARCH_ID, selector: `#${SEARCH_RESULTS_ADVANCED_SEARCH_ID}`, - title: 'Dive Deeper with Advanced Search 💪', + title: '💪 Dive deeper with advanced filters', content: ( - Use Advanced Search to find specific assets using granular filter predicates. + Advanced Filters offer additional capabilities to create more specific search queries. + + ), + }, + { + id: SEARCH_RESULTS_BROWSE_SIDEBAR_ID, + selector: `#${SEARCH_RESULTS_BROWSE_SIDEBAR_ID}`, + title: '🧭 Explore and refine your search by platform', + style: { minWidth: '425px' }, + content: ( + + Have a clear idea of the schema or folder you're searching for? Easily navigate your + organization's platforms inline. Then select a specific container you want to filter your results + by. + + ), + }, + { + id: SEARCH_RESULTS_FILTERS_V2_INTRO, + prerequisiteStepId: SEARCH_RESULTS_FILTERS_ID, + selector: `#${SEARCH_RESULTS_FILTERS_V2_INTRO}`, + title: 'Filters Have Moved', + content: ( + + Quickly find relevant assets with our new and improved filter interface! Our latest update has relocated + filters to the top of the screen for ease of access. ), }, diff --git a/datahub-web-react/src/app/onboarding/utils.tsx b/datahub-web-react/src/app/onboarding/utils.tsx index 202760bc1ba11..84bb7a0913038 100644 --- a/datahub-web-react/src/app/onboarding/utils.tsx +++ b/datahub-web-react/src/app/onboarding/utils.tsx @@ -15,6 +15,46 @@ export function getStepIds(userUrn: string) { return OnboardingConfig.map((step) => `${userUrn}-${step.id}`); } +// if the user just saw the prerequisiteStepId (in stepIdsToAdd) of a conditional step, we need to add this conditional step +export function getConditionalStepIdsToAdd(providedStepIds: string[], stepIdsToAdd: string[]) { + const conditionalStepIds: string[] = []; + + const providedSteps = providedStepIds + .map((stepId) => OnboardingConfig.find((step: OnboardingStep) => step.id === stepId)) + .filter((step) => !!step); + + providedSteps.forEach((step) => { + if ( + step?.prerequisiteStepId && + stepIdsToAdd.includes(step?.prerequisiteStepId) && + !stepIdsToAdd.includes(step.id || '') + ) { + conditionalStepIds.push(step.id || ''); + } + }); + + return conditionalStepIds; +} + +function hasStepBeenSeen(stepId: string, userUrn: string, educationSteps: StepStateResult[]) { + const convertedStepId = convertStepId(stepId, userUrn); + return educationSteps.some((step) => step.id === convertedStepId); +} + +export function hasSeenPrerequisiteStepIfExists( + step: OnboardingStep, + userUrn: string, + educationSteps: StepStateResult[], +) { + if (step?.prerequisiteStepId) { + if (hasStepBeenSeen(step.prerequisiteStepId, userUrn, educationSteps)) { + return true; + } + return false; + } + return true; +} + const StepTitle = styled(Typography.Title)` margin-botton: 5px; `; @@ -28,15 +68,14 @@ export function getStepsToRender( if (!educationSteps) return []; const filteredStepIds: string[] = reshow ? stepIds - : stepIds.filter((stepId) => { - const convertedStepId = convertStepId(stepId, userUrn); - // if we don't have this step in our educationSteps from GMS we haven't seen it yet - return !educationSteps.find((step) => step.id === convertedStepId); - }); + : stepIds.filter((stepId) => !hasStepBeenSeen(stepId, userUrn, educationSteps)); + + const finalStepIds = [...filteredStepIds]; - return filteredStepIds + return finalStepIds .map((stepId) => OnboardingConfig.find((step: OnboardingStep) => step.id === stepId)) .filter((step) => !!step) + .filter((step) => hasSeenPrerequisiteStepIfExists(step as OnboardingStep, userUrn, educationSteps)) // if this is a conditional step, only keep it if the prerequisite step has been seen .map((step) => ({ ...step, content: ( diff --git a/datahub-web-react/src/app/search/AdvancedSearchAddFilterSelect.tsx b/datahub-web-react/src/app/search/AdvancedSearchAddFilterSelect.tsx index 21cde1b1002cc..753742b6e3cb1 100644 --- a/datahub-web-react/src/app/search/AdvancedSearchAddFilterSelect.tsx +++ b/datahub-web-react/src/app/search/AdvancedSearchAddFilterSelect.tsx @@ -10,16 +10,23 @@ const StyledPlus = styled(PlusOutlined)` margin-right: 6px; `; -const selectStyle = { padding: 6, fontWeight: 500, width: 'auto' }; - interface Props { selectedFilters: Array; onFilterFieldSelect: (value) => void; + isCompact?: boolean; } const { Option } = Select; -export const AdvancedSearchAddFilterSelect = ({ selectedFilters, onFilterFieldSelect }: Props) => { +export const AdvancedSearchAddFilterSelect = ({ selectedFilters, onFilterFieldSelect, isCompact }: Props) => { + const selectStyle = { + padding: isCompact ? 3 : 6, + fontWeight: 500, + width: 'auto', + paddingTop: isCompact ? 0 : undefined, + paddingBottom: isCompact ? 0 : undefined, + }; + return ( onSelectOwnershipTypeActor(asset)} + onDeselect={(asset: any) => onDeselectOwnershipTypeActor(asset)} + tagRender={(tagProps) => { + return ( + + {ownershipTypesMap[tagProps.value.toString()]} + + ); + }} + > + {ownershipTypes.map((resOwnershipType) => { + return ( + + {resOwnershipType?.info?.name} + + ); + })} + + + )} )} Users}> diff --git a/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx b/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx index da599f53c3b50..68e91983babdb 100644 --- a/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx +++ b/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx @@ -103,6 +103,22 @@ export default function PolicyDetailsModal({ policy, visible, onClose, privilege ); }; + const resourceOwnersField = (actors) => { + if (!actors?.resourceOwners) { + return No; + } + if ((actors?.resolvedOwnershipTypes?.length ?? 0) > 0) { + return ( +
+ {actors?.resolvedOwnershipTypes?.map((type) => ( + {type.info.name} + ))} +
+ ); + } + return Yes - All owners; + }; + return ( @@ -180,7 +196,7 @@ export default function PolicyDetailsModal({ policy, visible, onClose, privilege
Applies to Owners - {policy?.actors?.resourceOwners ? 'True' : 'False'} + {resourceOwnersField(policy?.actors)}
Applies to Users diff --git a/datahub-web-react/src/graphql/policy.graphql b/datahub-web-react/src/graphql/policy.graphql index cb2b66ffc61a5..4ff25d5aa946b 100644 --- a/datahub-web-react/src/graphql/policy.graphql +++ b/datahub-web-react/src/graphql/policy.graphql @@ -34,6 +34,13 @@ query listPolicies($input: ListPoliciesInput!) { allUsers allGroups resourceOwners + resourceOwnersTypes + resolvedOwnershipTypes { + urn + info { + name + } + } resolvedUsers { username urn diff --git a/metadata-models/src/main/pegasus/com/linkedin/policy/DataHubActorFilter.pdl b/metadata-models/src/main/pegasus/com/linkedin/policy/DataHubActorFilter.pdl index 307ba2ba3d09a..5fcb0819d9f69 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/policy/DataHubActorFilter.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/policy/DataHubActorFilter.pdl @@ -1,6 +1,7 @@ namespace com.linkedin.policy import com.linkedin.common.Urn +import com.linkedin.ownership.OwnershipTypeInfo /** * Information used to filter DataHub actors. @@ -23,6 +24,11 @@ record DataHubActorFilter { */ resourceOwners: boolean = false + /** + * Define type of ownership for the policy + */ + resourceOwnersTypes: optional array[Urn] + /** * Whether the filter should apply to all users. */ diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java index 44b0c0d56f6fe..6a36fac7de4e0 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java @@ -2,15 +2,19 @@ import com.datahub.authentication.Authentication; import com.google.common.collect.ImmutableSet; +import com.linkedin.common.Owner; +import com.linkedin.common.Ownership; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.StringArray; import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.client.EntityClient; import com.linkedin.identity.GroupMembership; import com.linkedin.identity.NativeGroupMembership; import com.linkedin.identity.RoleMembership; +import com.linkedin.metadata.Constants; import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.policy.DataHubActorFilter; import com.linkedin.policy.DataHubPolicyInfo; @@ -28,6 +32,7 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -307,11 +312,34 @@ private boolean isOwnerMatch( if (!actorFilter.isResourceOwners() || !requestResource.isPresent()) { return false; } - return isActorOwner(actor, requestResource.get(), context); + List ownershipTypes = actorFilter.getResourceOwnersTypes(); + return isActorOwner(actor, requestResource.get(), ownershipTypes, context); } - private boolean isActorOwner(Urn actor, ResolvedResourceSpec resourceSpec, PolicyEvaluationContext context) { - Set owners = resourceSpec.getOwners(); + private Set getOwnersForType(ResourceSpec resourceSpec, List ownershipTypes) { + Urn entityUrn = UrnUtils.getUrn(resourceSpec.getResource()); + EnvelopedAspect ownershipAspect; + try { + EntityResponse response = _entityClient.getV2(entityUrn.getEntityType(), entityUrn, + Collections.singleton(Constants.OWNERSHIP_ASPECT_NAME), _systemAuthentication); + if (response == null || !response.getAspects().containsKey(Constants.OWNERSHIP_ASPECT_NAME)) { + return Collections.emptySet(); + } + ownershipAspect = response.getAspects().get(Constants.OWNERSHIP_ASPECT_NAME); + } catch (Exception e) { + log.error("Error while retrieving ownership aspect for urn {}", entityUrn, e); + return Collections.emptySet(); + } + Ownership ownership = new Ownership(ownershipAspect.getValue().data()); + Stream ownersStream = ownership.getOwners().stream(); + if (ownershipTypes != null) { + ownersStream = ownersStream.filter(owner -> ownershipTypes.contains(owner.getTypeUrn())); + } + return ownersStream.map(owner -> owner.getOwner().toString()).collect(Collectors.toSet()); + } + + private boolean isActorOwner(Urn actor, ResolvedResourceSpec resourceSpec, List ownershipTypes, PolicyEvaluationContext context) { + Set owners = this.getOwnersForType(resourceSpec.getSpec(), ownershipTypes); if (isUserOwner(actor, owners)) { return true; } diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java index 732b7633aad1a..99d8fee309d91 100644 --- a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java +++ b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java @@ -21,6 +21,7 @@ import com.linkedin.identity.CorpUserInfo; import com.linkedin.identity.GroupMembership; import com.linkedin.identity.RoleMembership; +import com.linkedin.metadata.Constants; import com.linkedin.policy.DataHubActorFilter; import com.linkedin.policy.DataHubPolicyInfo; import com.linkedin.policy.DataHubResourceFilter; @@ -51,6 +52,10 @@ public class PolicyEngineTest { private static final String DOMAIN_URN = "urn:li:domain:domain1"; + private static final String OWNERSHIP_TYPE_URN = "urn:li:ownershipType:__system__technical_owner"; + + private static final String OTHER_OWNERSHIP_TYPE_URN = "urn:li:ownershipType:__system__data_steward"; + private EntityClient _entityClient; private PolicyEngine _policyEngine; @@ -507,6 +512,13 @@ public void testEvaluatePolicyActorFilterUserResourceOwnersMatch() throws Except resourceFilter.setType("dataset"); dataHubPolicyInfo.setResources(resourceFilter); + final EntityResponse entityResponse = new EntityResponse(); + final EnvelopedAspectMap aspectMap = new EnvelopedAspectMap(); + aspectMap.put(OWNERSHIP_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(createOwnershipAspect(true, false).data()))); + entityResponse.setAspects(aspectMap); + when(_entityClient.getV2(eq(resourceUrn.getEntityType()), eq(resourceUrn), eq(Collections.singleton(Constants.OWNERSHIP_ASPECT_NAME)), + any())).thenReturn(entityResponse); + ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN, ImmutableSet.of(AUTHORIZED_PRINCIPAL), Collections.emptySet()); // Assert authorized user can edit entity tags, because he is a user owner. @@ -520,6 +532,84 @@ public void testEvaluatePolicyActorFilterUserResourceOwnersMatch() throws Except eq(null), any()); } + @Test + public void testEvaluatePolicyActorFilterUserResourceOwnersTypeMatch() throws Exception { + + final DataHubPolicyInfo dataHubPolicyInfo = new DataHubPolicyInfo(); + dataHubPolicyInfo.setType(METADATA_POLICY_TYPE); + dataHubPolicyInfo.setState(ACTIVE_POLICY_STATE); + dataHubPolicyInfo.setPrivileges(new StringArray("EDIT_ENTITY_TAGS")); + dataHubPolicyInfo.setDisplayName("My Test Display"); + dataHubPolicyInfo.setDescription("My test display!"); + dataHubPolicyInfo.setEditable(true); + + final DataHubActorFilter actorFilter = new DataHubActorFilter(); + actorFilter.setResourceOwners(true); + actorFilter.setAllUsers(false); + actorFilter.setAllGroups(false); + actorFilter.setResourceOwnersTypes(new UrnArray(ImmutableList.of(Urn.createFromString(OWNERSHIP_TYPE_URN)))); + dataHubPolicyInfo.setActors(actorFilter); + + final DataHubResourceFilter resourceFilter = new DataHubResourceFilter(); + resourceFilter.setAllResources(true); + resourceFilter.setType("dataset"); + dataHubPolicyInfo.setResources(resourceFilter); + + final EntityResponse entityResponse = new EntityResponse(); + final EnvelopedAspectMap aspectMap = new EnvelopedAspectMap(); + aspectMap.put(OWNERSHIP_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(createOwnershipAspectWithTypeUrn(OWNERSHIP_TYPE_URN).data()))); + entityResponse.setAspects(aspectMap); + when(_entityClient.getV2(eq(resourceUrn.getEntityType()), eq(resourceUrn), eq(Collections.singleton(Constants.OWNERSHIP_ASPECT_NAME)), + any())).thenReturn(entityResponse); + + ResolvedResourceSpec resourceSpec = + buildResourceResolvers("dataset", RESOURCE_URN, ImmutableSet.of(AUTHORIZED_PRINCIPAL), Collections.emptySet()); + + PolicyEngine.PolicyEvaluationResult result1 = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS", + Optional.of(resourceSpec)); + assertTrue(result1.isGranted()); + } + + @Test + public void testEvaluatePolicyActorFilterUserResourceOwnersTypeNoMatch() throws Exception { + + final DataHubPolicyInfo dataHubPolicyInfo = new DataHubPolicyInfo(); + dataHubPolicyInfo.setType(METADATA_POLICY_TYPE); + dataHubPolicyInfo.setState(ACTIVE_POLICY_STATE); + dataHubPolicyInfo.setPrivileges(new StringArray("EDIT_ENTITY_TAGS")); + dataHubPolicyInfo.setDisplayName("My Test Display"); + dataHubPolicyInfo.setDescription("My test display!"); + dataHubPolicyInfo.setEditable(true); + + final DataHubActorFilter actorFilter = new DataHubActorFilter(); + actorFilter.setResourceOwners(true); + actorFilter.setAllUsers(false); + actorFilter.setAllGroups(false); + actorFilter.setResourceOwnersTypes(new UrnArray(ImmutableList.of(Urn.createFromString(OWNERSHIP_TYPE_URN)))); + dataHubPolicyInfo.setActors(actorFilter); + + final DataHubResourceFilter resourceFilter = new DataHubResourceFilter(); + resourceFilter.setAllResources(true); + resourceFilter.setType("dataset"); + dataHubPolicyInfo.setResources(resourceFilter); + + final EntityResponse entityResponse = new EntityResponse(); + final EnvelopedAspectMap aspectMap = new EnvelopedAspectMap(); + aspectMap.put(OWNERSHIP_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(createOwnershipAspectWithTypeUrn(OTHER_OWNERSHIP_TYPE_URN).data()))); + entityResponse.setAspects(aspectMap); + when(_entityClient.getV2(eq(resourceUrn.getEntityType()), eq(resourceUrn), eq(Collections.singleton(Constants.OWNERSHIP_ASPECT_NAME)), + any())).thenReturn(entityResponse); + + ResolvedResourceSpec resourceSpec = + buildResourceResolvers("dataset", RESOURCE_URN, ImmutableSet.of(AUTHORIZED_PRINCIPAL), Collections.emptySet()); + + PolicyEngine.PolicyEvaluationResult result1 = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS", + Optional.of(resourceSpec)); + assertFalse(result1.isGranted()); + } + @Test public void testEvaluatePolicyActorFilterGroupResourceOwnersMatch() throws Exception { @@ -542,6 +632,13 @@ public void testEvaluatePolicyActorFilterGroupResourceOwnersMatch() throws Excep resourceFilter.setType("dataset"); dataHubPolicyInfo.setResources(resourceFilter); + final EntityResponse entityResponse = new EntityResponse(); + final EnvelopedAspectMap aspectMap = new EnvelopedAspectMap(); + aspectMap.put(OWNERSHIP_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(createOwnershipAspect(false, true).data()))); + entityResponse.setAspects(aspectMap); + when(_entityClient.getV2(eq(resourceUrn.getEntityType()), eq(resourceUrn), eq(Collections.singleton(Constants.OWNERSHIP_ASPECT_NAME)), + any())).thenReturn(entityResponse); + ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN, ImmutableSet.of(AUTHORIZED_GROUP), Collections.emptySet()); // Assert authorized user can edit entity tags, because he is a user owner. @@ -905,6 +1002,12 @@ public void testGetGrantedPrivileges() throws Exception { _policyEngine.getGrantedPrivileges(policies, UrnUtils.getUrn(AUTHORIZED_PRINCIPAL), Optional.of(resourceSpec)), ImmutableList.of("PRIVILEGE_1")); + final EntityResponse entityResponse = new EntityResponse(); + final EnvelopedAspectMap aspectMap = new EnvelopedAspectMap(); + aspectMap.put(OWNERSHIP_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(createOwnershipAspect(true, false).data()))); + entityResponse.setAspects(aspectMap); + when(_entityClient.getV2(eq(resourceUrn.getEntityType()), eq(resourceUrn), eq(Collections.singleton(Constants.OWNERSHIP_ASPECT_NAME)), + any())).thenReturn(entityResponse); resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN, Collections.singleton(AUTHORIZED_PRINCIPAL), Collections.singleton(DOMAIN_URN)); // Is owner assertEquals( @@ -1034,6 +1137,20 @@ private Ownership createOwnershipAspect(final Boolean addUserOwner, final Boolea return ownershipAspect; } + private Ownership createOwnershipAspectWithTypeUrn(final String typeUrn) throws Exception { + final Ownership ownershipAspect = new Ownership(); + final OwnerArray owners = new OwnerArray(); + + final Owner userOwner = new Owner(); + userOwner.setOwner(Urn.createFromString(AUTHORIZED_PRINCIPAL)); + userOwner.setTypeUrn(Urn.createFromString(typeUrn)); + owners.add(userOwner); + + ownershipAspect.setOwners(owners); + ownershipAspect.setLastModified(new AuditStamp().setTime(0).setActor(Urn.createFromString("urn:li:corpuser:foo"))); + return ownershipAspect; + } + private EntityResponse createAuthorizedEntityResponse() throws URISyntaxException { final EntityResponse entityResponse = new EntityResponse(); final EnvelopedAspectMap aspectMap = new EnvelopedAspectMap(); diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json index e0ea8ed8bd81b..83ecaf41022c4 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json @@ -5287,6 +5287,14 @@ "type" : "boolean", "doc" : "Whether the filter should return true for owners of a particular resource.\nOnly applies to policies of type 'Metadata', which have a resource associated with them.", "default" : false + }, { + "name" : "resourceOwnersTypes", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.Urn" + }, + "doc" : "Define type of ownership for the policy", + "optional" : true }, { "name" : "allUsers", "type" : "boolean", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json index dceb4ef1c1d75..2676c2687bd72 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json @@ -5281,6 +5281,14 @@ "type" : "boolean", "doc" : "Whether the filter should return true for owners of a particular resource.\nOnly applies to policies of type 'Metadata', which have a resource associated with them.", "default" : false + }, { + "name" : "resourceOwnersTypes", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.Urn" + }, + "doc" : "Define type of ownership for the policy", + "optional" : true }, { "name" : "allUsers", "type" : "boolean", From 74ab1bea06cbbde52ed5a969d8c0e957b7c9c246 Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Tue, 4 Jul 2023 19:19:39 +0200 Subject: [PATCH 093/222] fix(ingest/s3): Fix for flaky s3 test - uploading s3 files in consistent order (#8367) --- metadata-ingestion/tests/integration/s3/test_s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/tests/integration/s3/test_s3.py b/metadata-ingestion/tests/integration/s3/test_s3.py index c896c8d60cfc6..645593b1a46bb 100644 --- a/metadata-ingestion/tests/integration/s3/test_s3.py +++ b/metadata-ingestion/tests/integration/s3/test_s3.py @@ -56,7 +56,7 @@ def s3_populate(pytestconfig, s3_resource, s3_client, bucket_names): pytestconfig.rootpath / "tests/integration/s3/test_data/local_system/" ) for root, _dirs, files in os.walk(test_resources_dir): - for file in files: + for file in sorted(files): full_path = os.path.join(root, file) rel_path = os.path.relpath(full_path, test_resources_dir) bkt.upload_file(full_path, rel_path) From 5901e94ee054790df2ab61490d41890e3ac9bf88 Mon Sep 17 00:00:00 2001 From: Francesco Macagno Date: Wed, 5 Jul 2023 04:56:59 -0500 Subject: [PATCH 094/222] fix(ingest/airflow): Remove info log on import (#8246) Co-authored-by: Tamas Nemeth --- metadata-ingestion/src/datahub_provider/_plugin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/metadata-ingestion/src/datahub_provider/_plugin.py b/metadata-ingestion/src/datahub_provider/_plugin.py index 324b292685222..b4ac5a36c6eae 100644 --- a/metadata-ingestion/src/datahub_provider/_plugin.py +++ b/metadata-ingestion/src/datahub_provider/_plugin.py @@ -351,8 +351,6 @@ def _patch_policy(settings): def _patch_datahub_policy(): - logger.info("Patching datahub policy") - with contextlib.suppress(ImportError): import airflow_local_settings From 0f2a21df4e36d7b533ef8780db2b671358405680 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Wed, 5 Jul 2023 13:11:44 -0400 Subject: [PATCH 095/222] fix(ui) Update copy of the demo site acryl banner (#8370) --- datahub-web-react/src/app/home/AcrylDemoBanner.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datahub-web-react/src/app/home/AcrylDemoBanner.tsx b/datahub-web-react/src/app/home/AcrylDemoBanner.tsx index d2e79d4c21cae..87efae03e0a7c 100644 --- a/datahub-web-react/src/app/home/AcrylDemoBanner.tsx +++ b/datahub-web-react/src/app/home/AcrylDemoBanner.tsx @@ -40,7 +40,7 @@ export default function AcrylDemoBanner() { Schedule a Demo of Managed Datahub - DataHub is already the industries #1 Open Source Data Catalog.{' '} + DataHub is already the industry's #1 Open Source Data Catalog.{' '} Schedule a demo {' '} - of Acryl DataHub to see the advanced features that take it to the next level! + of Acryl Cloud to see the advanced features that take it to the next level! From afbd52bdf0d84515e555eb2e1eeea01fc47690dc Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Wed, 5 Jul 2023 13:41:24 -0400 Subject: [PATCH 096/222] test(ingest/mysql): Configure sql_server tests for arm64 (#8360) Co-authored-by: Tamas Nemeth --- .../tests/integration/sql_server/docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/tests/integration/sql_server/docker-compose.yml b/metadata-ingestion/tests/integration/sql_server/docker-compose.yml index f6c55a5248a8f..2ce0433140554 100644 --- a/metadata-ingestion/tests/integration/sql_server/docker-compose.yml +++ b/metadata-ingestion/tests/integration/sql_server/docker-compose.yml @@ -1,7 +1,8 @@ -version: "2" +version: "3" services: testsqlserver: image: "mcr.microsoft.com/mssql/server:latest" + platform: linux/amd64 container_name: "testsqlserver" environment: ACCEPT_EULA: "Y" From 3207e732da4bd75675dc27818eb3cc8f9acbfde2 Mon Sep 17 00:00:00 2001 From: Joshua Eilers Date: Wed, 5 Jul 2023 13:39:00 -0700 Subject: [PATCH 097/222] fix(browse): filter entities by whether they might exist in the instance (#8355) --- .../src/app/search/sidebar/BrowseSidebar.tsx | 8 ++--- .../src/app/search/sidebar/EntityNode.tsx | 4 +-- .../app/search/sidebar/EnvironmentNode.tsx | 4 +-- .../search/sidebar/useAggregationsQuery.ts | 17 +++++++--- .../app/search/sidebar/useSidebarEntities.ts | 34 +++++++++++++++++++ 5 files changed, 53 insertions(+), 14 deletions(-) create mode 100644 datahub-web-react/src/app/search/sidebar/useSidebarEntities.ts diff --git a/datahub-web-react/src/app/search/sidebar/BrowseSidebar.tsx b/datahub-web-react/src/app/search/sidebar/BrowseSidebar.tsx index c61f1c8ce5fa6..64aa4e70cb117 100644 --- a/datahub-web-react/src/app/search/sidebar/BrowseSidebar.tsx +++ b/datahub-web-react/src/app/search/sidebar/BrowseSidebar.tsx @@ -3,10 +3,9 @@ import styled from 'styled-components'; import { Typography } from 'antd'; import EntityNode from './EntityNode'; import { BrowseProvider } from './BrowseContext'; -import useAggregationsQuery from './useAggregationsQuery'; -import { ENTITY_FILTER_NAME } from '../utils/constants'; import SidebarLoadingError from './SidebarLoadingError'; import { SEARCH_RESULTS_BROWSE_SIDEBAR_ID } from '../../onboarding/config/SearchOnboardingConfig'; +import useSidebarEntities from './useSidebarEntities'; const Sidebar = styled.div<{ visible: boolean; width: number }>` height: 100%; @@ -40,9 +39,8 @@ type Props = { }; const BrowseSidebar = ({ visible, width }: Props) => { - const { error, entityAggregations } = useAggregationsQuery({ + const { error, entityAggregations, retry } = useSidebarEntities({ skip: !visible, - facets: [ENTITY_FILTER_NAME], }); return ( @@ -57,7 +55,7 @@ const BrowseSidebar = ({ visible, width }: Props) => { ))} - {error && } + {error && } ); diff --git a/datahub-web-react/src/app/search/sidebar/EntityNode.tsx b/datahub-web-react/src/app/search/sidebar/EntityNode.tsx index 9615f53347b53..46310d4534f46 100644 --- a/datahub-web-react/src/app/search/sidebar/EntityNode.tsx +++ b/datahub-web-react/src/app/search/sidebar/EntityNode.tsx @@ -41,7 +41,7 @@ const EntityNode = () => { if (count) toggle(); }; - const { loaded, error, environmentAggregations, platformAggregations } = useAggregationsQuery({ + const { loaded, error, environmentAggregations, platformAggregations, retry } = useAggregationsQuery({ skip: !isOpen, facets: [ORIGIN_FILTER_NAME, PLATFORM_FILTER_NAME], }); @@ -93,7 +93,7 @@ const EntityNode = () => { ))} - {error && } + {error && } } /> diff --git a/datahub-web-react/src/app/search/sidebar/EnvironmentNode.tsx b/datahub-web-react/src/app/search/sidebar/EnvironmentNode.tsx index de01e13187910..1673f497696ba 100644 --- a/datahub-web-react/src/app/search/sidebar/EnvironmentNode.tsx +++ b/datahub-web-react/src/app/search/sidebar/EnvironmentNode.tsx @@ -38,7 +38,7 @@ const EnvironmentNode = () => { if (count) toggle(); }; - const { loaded, error, platformAggregations } = useAggregationsQuery({ + const { loaded, error, platformAggregations, retry } = useAggregationsQuery({ skip: !isOpen, facets: [PLATFORM_FILTER_NAME], }); @@ -75,7 +75,7 @@ const EnvironmentNode = () => { ))} - {error && } + {error && } } /> diff --git a/datahub-web-react/src/app/search/sidebar/useAggregationsQuery.ts b/datahub-web-react/src/app/search/sidebar/useAggregationsQuery.ts index a9dd383e192c1..c32dca8ba0537 100644 --- a/datahub-web-react/src/app/search/sidebar/useAggregationsQuery.ts +++ b/datahub-web-react/src/app/search/sidebar/useAggregationsQuery.ts @@ -9,9 +9,10 @@ import { useSidebarFilters } from './useSidebarFilters'; type Props = { facets: string[]; skip: boolean; + excludeFilters?: boolean; }; -const useAggregationsQuery = ({ facets, skip }: Props) => { +const useAggregationsQuery = ({ facets, excludeFilters = false, skip }: Props) => { const registry = useEntityRegistry(); const sidebarFilters = useSidebarFilters(); @@ -20,16 +21,17 @@ const useAggregationsQuery = ({ facets, skip }: Props) => { previousData, loading, error, + refetch, } = useAggregateAcrossEntitiesQuery({ skip, fetchPolicy: 'cache-first', variables: { input: { - types: sidebarFilters.entityFilters, + ...(excludeFilters ? {} : { types: sidebarFilters.entityFilters }), facets, - orFilters: sidebarFilters.orFilters, - viewUrn: sidebarFilters.viewUrn, - query: sidebarFilters.query, + ...(excludeFilters ? {} : { orFilters: sidebarFilters.orFilters }), + ...(excludeFilters ? {} : { viewUrn: sidebarFilters.viewUrn }), + query: excludeFilters ? '*' : sidebarFilters.query, searchFlags: { maxAggValues: MAX_AGGREGATION_VALUES, }, @@ -37,6 +39,10 @@ const useAggregationsQuery = ({ facets, skip }: Props) => { }, }); + const retry = () => { + if (refetch) refetch(); + }; + // This approach of falling back to previousData is needed to avoid a full re-mount of the sidebar entities const data = error ? null : newData ?? previousData; const loaded = !!data || !!error; @@ -74,6 +80,7 @@ const useAggregationsQuery = ({ facets, skip }: Props) => { entityAggregations, environmentAggregations, platformAggregations, + retry, } as const; }; diff --git a/datahub-web-react/src/app/search/sidebar/useSidebarEntities.ts b/datahub-web-react/src/app/search/sidebar/useSidebarEntities.ts new file mode 100644 index 0000000000000..08150a671a9c5 --- /dev/null +++ b/datahub-web-react/src/app/search/sidebar/useSidebarEntities.ts @@ -0,0 +1,34 @@ +import { useMemo } from 'react'; +import { ENTITY_FILTER_NAME } from '../utils/constants'; +import useAggregationsQuery from './useAggregationsQuery'; + +type Props = { + skip: boolean; +}; + +const useSidebarEntities = ({ skip }: Props) => { + const { + error: filteredError, + entityAggregations: filteredAggs, + retry: retryFilteredAggs, + } = useAggregationsQuery({ + skip, + facets: [ENTITY_FILTER_NAME], + }); + + const { error: baseError, entityAggregations: baseAggs } = useAggregationsQuery({ + skip, + facets: [ENTITY_FILTER_NAME], + excludeFilters: true, + }); + + const result = useMemo(() => { + if (filteredError || baseError) return filteredAggs; // Fallback to filtered aggs on any error + if (!filteredAggs || !baseAggs) return null; // If we're loading one of the queries, wait to render + return filteredAggs.filter((agg) => baseAggs.some((base) => base.value === agg.value && !!base.count)); + }, [baseAggs, baseError, filteredAggs, filteredError]); + + return { error: filteredError, entityAggregations: result, retry: retryFilteredAggs } as const; +}; + +export default useSidebarEntities; From 393b5a30393129654dfddb71135ed84e834f297f Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 5 Jul 2023 15:21:27 -0700 Subject: [PATCH 098/222] ci(docs): add missing deps for lxml package for vercel (#8372) --- metadata-ingestion/scripts/install_deps.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/scripts/install_deps.sh b/metadata-ingestion/scripts/install_deps.sh index 8aa0f44ce7686..7e6b6956d8bb8 100755 --- a/metadata-ingestion/scripts/install_deps.sh +++ b/metadata-ingestion/scripts/install_deps.sh @@ -16,7 +16,9 @@ else cyrus-sasl-devel \ openldap-clients \ sqlite-devel \ - xz-devel + xz-devel \ + libxml2-devel \ + libxslt-devel else $sudo_cmd apt-get update && $sudo_cmd apt-get install -y \ librdkafka-dev \ From a5f420ffd4ccf2e2765391d74f44277d96f51a8e Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Thu, 6 Jul 2023 10:42:07 -0500 Subject: [PATCH 099/222] feat(browsepathv2): enable incremental update browsepath (#8354) --- .../elasticsearch/query/ESBrowseDAO.java | 2 +- .../query/request/SearchRequestHandler.java | 4 +- .../metadata/search/utils/ESUtils.java | 59 +++++++++-------- .../request/SearchRequestHandlerTest.java | 63 +++++++++++++++---- .../boot/steps/BackfillBrowsePathsV2Step.java | 46 +++++++++++--- .../steps/BackfillBrowsePathsV2StepTest.java | 5 +- .../e2e/mutations/managed_ingestion.js | 2 +- 7 files changed, 128 insertions(+), 53 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java index 646dc2e4ef0a8..3cb3c441afd68 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java @@ -437,7 +437,7 @@ private QueryBuilder buildQueryStringV2(@Nonnull String entityName, @Nonnull Str queryBuilder.filter(QueryBuilders.rangeQuery(BROWSE_PATH_V2_DEPTH).gt(browseDepthVal)); - queryBuilder.must(SearchRequestHandler.getFilterQuery(filter)); + queryBuilder.filter(SearchRequestHandler.getFilterQuery(filter)); return queryBuilder; } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java index 09fc210583e28..21cec705d57f3 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java @@ -191,7 +191,7 @@ public SearchRequest getSearchRequest(@Nonnull String input, @Nullable Filter fi BoolQueryBuilder filterQuery = getFilterQuery(filter); searchSourceBuilder.query(QueryBuilders.boolQuery() .must(getQuery(input, finalSearchFlags.isFulltext())) - .must(filterQuery)); + .filter(filterQuery)); if (!finalSearchFlags.isSkipAggregates()) { _aggregationQueryBuilder.getAggregations(facets).forEach(searchSourceBuilder::aggregation); } @@ -231,7 +231,7 @@ public SearchRequest getSearchRequest(@Nonnull String input, @Nullable Filter fi searchSourceBuilder.fetchSource("urn", null); BoolQueryBuilder filterQuery = getFilterQuery(filter); - searchSourceBuilder.query(QueryBuilders.boolQuery().must(getQuery(input, finalSearchFlags.isFulltext())).must(filterQuery)); + searchSourceBuilder.query(QueryBuilders.boolQuery().must(getQuery(input, finalSearchFlags.isFulltext())).filter(filterQuery)); _aggregationQueryBuilder.getAggregations().forEach(searchSourceBuilder::aggregation); searchSourceBuilder.highlighter(getHighlights()); ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java index 2ab271e171b80..8a385e4ab2b54 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java @@ -109,10 +109,11 @@ public static BoolQueryBuilder buildConjunctiveFilterQuery(@Nonnull ConjunctiveC boolean isTimeseries) { final BoolQueryBuilder andQueryBuilder = new BoolQueryBuilder(); conjunctiveCriterion.getAnd().forEach(criterion -> { - if (!criterion.getValue().trim().isEmpty() || criterion.hasValues() - || criterion.getCondition() == Condition.IS_NULL) { + if (Set.of(Condition.EXISTS, Condition.IS_NULL).contains(criterion.getCondition()) + || !criterion.getValue().trim().isEmpty() || criterion.hasValues()) { if (!criterion.isNegated()) { - andQueryBuilder.must(getQueryBuilderFromCriterion(criterion, isTimeseries)); + // `filter` instead of `must` (enables caching and bypasses scoring) + andQueryBuilder.filter(getQueryBuilderFromCriterion(criterion, isTimeseries)); } else { andQueryBuilder.mustNot(getQueryBuilderFromCriterion(criterion, isTimeseries)); } @@ -261,8 +262,12 @@ private static QueryBuilder getQueryBuilderFromCriterionForFieldToExpand( Criterion criterionToQuery = new Criterion(); criterionToQuery.setCondition(criterion.getCondition()); criterionToQuery.setNegated(criterion.isNegated()); - criterionToQuery.setValue(criterion.getValue()); - criterionToQuery.setValues(criterion.getValues()); + if (criterion.hasValues()) { + criterionToQuery.setValues(criterion.getValues()); + } + if (criterion.hasValue()) { + criterionToQuery.setValue(criterion.getValue()); + } criterionToQuery.setField(toKeywordField(field, isTimeseries)); orQueryBuilder.should(getQueryBuilderFromCriterionForSingleField(criterionToQuery, isTimeseries)); } @@ -274,30 +279,32 @@ private static QueryBuilder getQueryBuilderFromCriterionForSingleField(@Nonnull final Condition condition = criterion.getCondition(); final String fieldName = toFacetField(criterion.getField()); - if (condition == Condition.EQUAL) { - return buildEqualsConditionFromCriterion(fieldName, criterion, isTimeseries); - } else if (condition == Condition.IS_NULL) { + if (condition == Condition.IS_NULL) { return QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(criterion.getField())).queryName(fieldName); } else if (condition == Condition.EXISTS) { return QueryBuilders.boolQuery().must(QueryBuilders.existsQuery(criterion.getField())).queryName(fieldName); - // TODO: Support multi-match on the following operators (using new 'values' field) - } else if (condition == Condition.GREATER_THAN) { - return QueryBuilders.rangeQuery(criterion.getField()).gt(criterion.getValue().trim()).queryName(fieldName); - } else if (condition == Condition.GREATER_THAN_OR_EQUAL_TO) { - return QueryBuilders.rangeQuery(criterion.getField()).gte(criterion.getValue().trim()).queryName(fieldName); - } else if (condition == Condition.LESS_THAN) { - return QueryBuilders.rangeQuery(criterion.getField()).lt(criterion.getValue().trim()).queryName(fieldName); - } else if (condition == Condition.LESS_THAN_OR_EQUAL_TO) { - return QueryBuilders.rangeQuery(criterion.getField()).lte(criterion.getValue().trim()).queryName(fieldName); - } else if (condition == Condition.CONTAIN) { - return QueryBuilders.wildcardQuery(toKeywordField(criterion.getField(), isTimeseries), - "*" + ESUtils.escapeReservedCharacters(criterion.getValue().trim()) + "*").queryName(fieldName); - } else if (condition == Condition.START_WITH) { - return QueryBuilders.wildcardQuery(toKeywordField(criterion.getField(), isTimeseries), - ESUtils.escapeReservedCharacters(criterion.getValue().trim()) + "*").queryName(fieldName); - } else if (condition == Condition.END_WITH) { - return QueryBuilders.wildcardQuery(toKeywordField(criterion.getField(), isTimeseries), - "*" + ESUtils.escapeReservedCharacters(criterion.getValue().trim())).queryName(fieldName); + } else if (criterion.hasValues() || criterion.hasValue()) { + if (condition == Condition.EQUAL) { + return buildEqualsConditionFromCriterion(fieldName, criterion, isTimeseries); + // TODO: Support multi-match on the following operators (using new 'values' field) + } else if (condition == Condition.GREATER_THAN) { + return QueryBuilders.rangeQuery(criterion.getField()).gt(criterion.getValue().trim()).queryName(fieldName); + } else if (condition == Condition.GREATER_THAN_OR_EQUAL_TO) { + return QueryBuilders.rangeQuery(criterion.getField()).gte(criterion.getValue().trim()).queryName(fieldName); + } else if (condition == Condition.LESS_THAN) { + return QueryBuilders.rangeQuery(criterion.getField()).lt(criterion.getValue().trim()).queryName(fieldName); + } else if (condition == Condition.LESS_THAN_OR_EQUAL_TO) { + return QueryBuilders.rangeQuery(criterion.getField()).lte(criterion.getValue().trim()).queryName(fieldName); + } else if (condition == Condition.CONTAIN) { + return QueryBuilders.wildcardQuery(toKeywordField(criterion.getField(), isTimeseries), + "*" + ESUtils.escapeReservedCharacters(criterion.getValue().trim()) + "*").queryName(fieldName); + } else if (condition == Condition.START_WITH) { + return QueryBuilders.wildcardQuery(toKeywordField(criterion.getField(), isTimeseries), + ESUtils.escapeReservedCharacters(criterion.getValue().trim()) + "*").queryName(fieldName); + } else if (condition == Condition.END_WITH) { + return QueryBuilders.wildcardQuery(toKeywordField(criterion.getField(), isTimeseries), + "*" + ESUtils.escapeReservedCharacters(criterion.getValue().trim())).queryName(fieldName); + } } throw new UnsupportedOperationException("Unsupported condition: " + condition); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandlerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandlerTest.java index 54b50e0788f09..095c99370e8d1 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandlerTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandlerTest.java @@ -29,6 +29,7 @@ import com.linkedin.metadata.query.filter.Filter; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.ExistsQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.MultiMatchQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; @@ -215,7 +216,7 @@ private BoolQueryBuilder constructFilterQuery(SearchRequestHandler requestHandle } private void testFilterQuery(BoolQueryBuilder testQuery) { - Optional mustNotHaveRemovedCondition = testQuery.must() + Optional mustNotHaveRemovedCondition = testQuery.filter() .stream() .filter(or -> or instanceof BoolQueryBuilder) .map(or -> (BoolQueryBuilder) or) @@ -292,13 +293,13 @@ public void testFilterFieldTagsByValues() { final BoolQueryBuilder testQuery = getQuery(filterCriterion); - // bool -> must -> [bool] -> should -> [bool] -> must -> [bool] -> should -> [terms] - List termsQueryBuilders = testQuery.must() + // bool -> filter -> [bool] -> should -> [bool] -> filter -> [bool] -> should -> [terms] + List termsQueryBuilders = testQuery.filter() .stream() .filter(or -> or instanceof BoolQueryBuilder) .flatMap(or -> ((BoolQueryBuilder) or).should().stream()) .filter(should -> should instanceof BoolQueryBuilder) - .flatMap(should -> ((BoolQueryBuilder) should).must().stream()) + .flatMap(should -> ((BoolQueryBuilder) should).filter().stream()) .filter(must -> must instanceof BoolQueryBuilder) .flatMap(must -> ((BoolQueryBuilder) must).should().stream()) .filter(should -> should instanceof TermsQueryBuilder) @@ -342,12 +343,12 @@ public void testFilterFieldTagsByValue() { final BoolQueryBuilder testQuery = getQuery(filterCriterion); // bool -> must -> [bool] -> should -> [bool] -> must -> [bool] -> should -> [bool] -> should -> [match] - List matchQueryBuilders = testQuery.must() + List matchQueryBuilders = testQuery.filter() .stream() .filter(or -> or instanceof BoolQueryBuilder) .flatMap(or -> ((BoolQueryBuilder) or).should().stream()) .filter(should -> should instanceof BoolQueryBuilder) - .flatMap(should -> ((BoolQueryBuilder) should).must().stream()) + .flatMap(should -> ((BoolQueryBuilder) should).filter().stream()) .filter(must -> must instanceof BoolQueryBuilder) .flatMap(must -> ((BoolQueryBuilder) must).should().stream()) .filter(should -> should instanceof BoolQueryBuilder) @@ -381,13 +382,13 @@ public void testFilterPlatformByValue() { final BoolQueryBuilder testQuery = getQuery(filterCriterion); - // bool -> must -> [bool] -> should -> [bool] -> must -> [bool] -> should -> [match] - List matchQueryBuilders = testQuery.must() + // bool -> filter -> [bool] -> should -> [bool] -> filter -> [bool] -> should -> [match] + List matchQueryBuilders = testQuery.filter() .stream() .filter(or -> or instanceof BoolQueryBuilder) .flatMap(or -> ((BoolQueryBuilder) or).should().stream()) .filter(should -> should instanceof BoolQueryBuilder) - .flatMap(should -> ((BoolQueryBuilder) should).must().stream()) + .flatMap(should -> ((BoolQueryBuilder) should).filter().stream()) .filter(must -> must instanceof BoolQueryBuilder) .flatMap(must -> ((BoolQueryBuilder) must).should().stream()) .filter(should -> should instanceof MultiMatchQueryBuilder) @@ -414,13 +415,13 @@ public void testFilterPlatformByValues() { final BoolQueryBuilder testQuery = getQuery(filterCriterion); - // bool -> must -> [bool] -> should -> [bool] -> must -> [terms] - List termsQueryBuilders = testQuery.must() + // bool -> filter -> [bool] -> should -> [bool] -> filter -> [terms] + List termsQueryBuilders = testQuery.filter() .stream() .filter(must -> must instanceof BoolQueryBuilder) .flatMap(must -> ((BoolQueryBuilder) must).should().stream()) .filter(should -> should instanceof BoolQueryBuilder) - .flatMap(should -> ((BoolQueryBuilder) should).must().stream()) + .flatMap(should -> ((BoolQueryBuilder) should).filter().stream()) .filter(must -> must instanceof TermsQueryBuilder) .map(must -> (TermsQueryBuilder) must) .collect(Collectors.toList()); @@ -439,6 +440,44 @@ public void testFilterPlatformByValues() { assertTrue(values.contains("bigquery")); } + @Test + public void testBrowsePathQueryFilter() { + // Condition: has `browsePaths` AND does NOT have `browsePathV2` + Criterion missingBrowsePathV2 = new Criterion(); + missingBrowsePathV2.setCondition(Condition.IS_NULL); + missingBrowsePathV2.setField("browsePathV2"); + // Excludes entities without browsePaths + Criterion hasBrowsePathV1 = new Criterion(); + hasBrowsePathV1.setCondition(Condition.EXISTS); + hasBrowsePathV1.setField("browsePaths"); + + CriterionArray criterionArray = new CriterionArray(); + criterionArray.add(missingBrowsePathV2); + criterionArray.add(hasBrowsePathV1); + + ConjunctiveCriterion conjunctiveCriterion = new ConjunctiveCriterion(); + conjunctiveCriterion.setAnd(criterionArray); + + ConjunctiveCriterionArray conjunctiveCriterionArray = new ConjunctiveCriterionArray(); + conjunctiveCriterionArray.add(conjunctiveCriterion); + + Filter filter = new Filter(); + filter.setOr(conjunctiveCriterionArray); + + BoolQueryBuilder test = SearchRequestHandler.getFilterQuery(filter); + + assertEquals(test.should().size(), 1); + + BoolQueryBuilder shouldQuery = (BoolQueryBuilder) test.should().get(0); + assertEquals(shouldQuery.filter().size(), 2); + + BoolQueryBuilder mustNotHaveV2 = (BoolQueryBuilder) shouldQuery.filter().get(0); + assertEquals(((ExistsQueryBuilder) mustNotHaveV2.mustNot().get(0)).fieldName(), "browsePathV2"); + + BoolQueryBuilder mustHaveV1 = (BoolQueryBuilder) shouldQuery.filter().get(1); + assertEquals(((ExistsQueryBuilder) mustHaveV1.must().get(0)).fieldName(), "browsePaths"); + } + private BoolQueryBuilder getQuery(final Criterion filterCriterion) { final Filter filter = new Filter().setOr( new ConjunctiveCriterionArray( diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java index 45e11d9d0b381..afe82f183b5de 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java @@ -9,6 +9,12 @@ import com.linkedin.metadata.Constants; import com.linkedin.metadata.boot.UpgradeStep; import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.search.ScrollResult; import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.search.SearchService; @@ -69,17 +75,39 @@ public void upgrade() throws Exception { } private String backfillBrowsePathsV2(String entityType, AuditStamp auditStamp, String scrollId) - throws Exception { + throws Exception { + + // Condition: has `browsePaths` AND does NOT have `browsePathV2` + Criterion missingBrowsePathV2 = new Criterion(); + missingBrowsePathV2.setCondition(Condition.IS_NULL); + missingBrowsePathV2.setField("browsePathV2"); + // Excludes entities without browsePaths + Criterion hasBrowsePathV1 = new Criterion(); + hasBrowsePathV1.setCondition(Condition.EXISTS); + hasBrowsePathV1.setField("browsePaths"); + + CriterionArray criterionArray = new CriterionArray(); + criterionArray.add(missingBrowsePathV2); + criterionArray.add(hasBrowsePathV1); + + ConjunctiveCriterion conjunctiveCriterion = new ConjunctiveCriterion(); + conjunctiveCriterion.setAnd(criterionArray); + + ConjunctiveCriterionArray conjunctiveCriterionArray = new ConjunctiveCriterionArray(); + conjunctiveCriterionArray.add(conjunctiveCriterion); + + Filter filter = new Filter(); + filter.setOr(conjunctiveCriterionArray); final ScrollResult scrollResult = _searchService.scrollAcrossEntities( - ImmutableList.of(entityType), - "*", - null, - null, - scrollId, - "5m", - BATCH_SIZE, - null + ImmutableList.of(entityType), + "*", + filter, + null, + scrollId, + "5m", + BATCH_SIZE, + null ); if (scrollResult.getNumEntities() == 0 || scrollResult.getEntities().size() == 0) { return null; diff --git a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2StepTest.java b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2StepTest.java index 8167f794369bd..f02b07f7a5fcb 100644 --- a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2StepTest.java +++ b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2StepTest.java @@ -14,6 +14,7 @@ import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.search.ScrollResult; import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.search.SearchEntityArray; @@ -88,7 +89,7 @@ public void testExecuteNoExistingBrowsePaths() throws Exception { Mockito.verify(mockSearchService, Mockito.times(9)).scrollAcrossEntities( Mockito.any(), Mockito.eq("*"), - Mockito.eq(null), + Mockito.any(Filter.class), Mockito.eq(null), Mockito.eq(null), Mockito.eq("5m"), @@ -157,7 +158,7 @@ private SearchService initMockSearchService() { Mockito.when(mockSearchService.scrollAcrossEntities( Mockito.eq(ImmutableList.of(ENTITY_TYPES.get(i))), Mockito.eq("*"), - Mockito.eq(null), + Mockito.any(Filter.class), Mockito.eq(null), Mockito.eq(null), Mockito.eq("5m"), diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js b/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js index 69c8968fd25c3..c92ba1f6e0df0 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js @@ -7,7 +7,7 @@ describe("run managed ingestion", () => { it("create run managed ingestion source", () => { let number = Math.floor(Math.random() * 100000); let testName = `cypress test source ${number}` - let cli_version = "0.9.3.3rc5"; + let cli_version = "0.10.4.3"; cy.login(); cy.goToIngestionPage(); cy.clickOptionWithText("Create new source"); From 91ebb0706a89ece386c248c7861d30ef6d03d79c Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Thu, 6 Jul 2023 10:42:47 -0500 Subject: [PATCH 100/222] chore(smoke-test): use a more recent ingestion cli version in tests (#8374) --- .../tests/cypress/cypress/e2e/mutations/managed_ingestion.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js b/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js index c92ba1f6e0df0..fd6703a7ef29e 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js @@ -37,4 +37,4 @@ describe("run managed ingestion", () => { cy.clickOptionWithText("Yes") cy.ensureTextNotPresent(testName) }) -}); \ No newline at end of file +}); From c57037eb86ea6c127f1d18d015de9720d2468697 Mon Sep 17 00:00:00 2001 From: Joshua Eilers Date: Thu, 6 Jul 2023 10:19:40 -0700 Subject: [PATCH 101/222] feat(stats): show size in bytes and scale at y=min (#8375) --- .../components/TimeSeriesChart.tsx | 39 +++++++++++++++--- .../Stats/historical/HistoricalStats.tsx | 32 +++++++++++++++ .../historical/charts/ProfilingRunsChart.tsx | 13 ++++++ .../Stats/historical/charts/StatChart.tsx | 41 +++++++++++-------- 4 files changed, 103 insertions(+), 22 deletions(-) diff --git a/datahub-web-react/src/app/analyticsDashboard/components/TimeSeriesChart.tsx b/datahub-web-react/src/app/analyticsDashboard/components/TimeSeriesChart.tsx index 796f68888caa0..6b9b808abfd0f 100644 --- a/datahub-web-react/src/app/analyticsDashboard/components/TimeSeriesChart.tsx +++ b/datahub-web-react/src/app/analyticsDashboard/components/TimeSeriesChart.tsx @@ -7,6 +7,15 @@ import Legend from './Legend'; import { addInterval } from '../../shared/time/timeUtils'; import { formatNumber } from '../../shared/formatNumber'; +type ScaleConfig = { + type: 'time' | 'timeUtc' | 'linear' | 'band' | 'ordinal'; + includeZero?: boolean; +}; + +type AxisConfig = { + formatter: (tick: number) => string; +}; + type Props = { chartData: TimeSeriesChartType; width: number; @@ -20,9 +29,16 @@ type Props = { crossHairLineColor?: string; }; insertBlankPoints?: boolean; + yScale?: ScaleConfig; + yAxis?: AxisConfig; }; -const MARGIN_SIZE = 40; +const MARGIN = { + TOP: 40, + RIGHT: 45, + BOTTOM: 40, + LEFT: 40, +}; function insertBlankAt(ts: number, newLine: Array) { const dateString = new Date(ts).toISOString(); @@ -60,7 +76,16 @@ export function computeLines(chartData: TimeSeriesChartType, insertBlankPoints: return returnLines; } -export const TimeSeriesChart = ({ chartData, width, height, hideLegend, style, insertBlankPoints }: Props) => { +export const TimeSeriesChart = ({ + chartData, + width, + height, + hideLegend, + style, + insertBlankPoints, + yScale, + yAxis, +}: Props) => { const ordinalColorScale = scaleOrdinal({ domain: chartData.lines.map((data) => data.name), range: lineColors.slice(0, chartData.lines.length), @@ -75,9 +100,13 @@ export const TimeSeriesChart = ({ chartData, width, height, hideLegend, style, i ariaLabel={chartData.title} width={width} height={height} - margin={{ top: MARGIN_SIZE, right: MARGIN_SIZE, bottom: MARGIN_SIZE, left: MARGIN_SIZE }} + margin={{ top: MARGIN.TOP, right: MARGIN.RIGHT, bottom: MARGIN.BOTTOM, left: MARGIN.LEFT }} xScale={{ type: 'time' }} - yScale={{ type: 'linear' }} + yScale={ + yScale ?? { + type: 'linear', + } + } renderTooltip={({ datum }) => (
{new Date(Number(datum.x)).toDateString()}
@@ -89,7 +118,7 @@ export const TimeSeriesChart = ({ chartData, width, height, hideLegend, style, i formatNumber(tick)} + tickFormat={(tick) => (yAxis?.formatter ? yAxis.formatter(tick) : formatNumber(tick))} /> {lines.map((line, i) => ( { + const formattedBytes = formatBytes(num); + return `${formattedBytes.number} ${formattedBytes.unit}`; + }; + + const placeholderChart = ( + + ); + const placeholderVerticalDivider = ( + + ); + return ( <> {profilesLoading && } @@ -216,6 +236,18 @@ export default function HistoricalStats({ urn, lookbackWindow }: Props) { values={columnCountChartValues} /> + + + + {placeholderVerticalDivider} + {placeholderChart} + diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/historical/charts/ProfilingRunsChart.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/historical/charts/ProfilingRunsChart.tsx index 9b5176f96b697..11ec85d65da42 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/historical/charts/ProfilingRunsChart.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/historical/charts/ProfilingRunsChart.tsx @@ -4,6 +4,7 @@ import styled from 'styled-components'; import { DatasetProfile } from '../../../../../../../../types.generated'; import ColumnStats from '../../snapshot/ColumnStats'; import TableStats from '../../snapshot/TableStats'; +import { formatBytes, formatNumberWithoutAbbreviation } from '../../../../../../../shared/formatNumber'; export const ChartTable = styled(Table)` margin-top: 16px; @@ -13,6 +14,12 @@ export type Props = { profiles: Array; }; +const bytesFormatter = (bytes: number) => { + const formattedBytes = formatBytes(bytes); + const fullBytes = formatNumberWithoutAbbreviation(bytes); + return `${formattedBytes.number} ${formattedBytes.unit} (${fullBytes} bytes)`; +}; + export default function ProfilingRunsChart({ profiles }: Props) { const [showModal, setShowModal] = useState(false); const [selectedProfileIndex, setSelectedProfileIndex] = useState(-1); @@ -33,6 +40,7 @@ export default function ProfilingRunsChart({ profiles }: Props) { timestamp: `${profileDate.toLocaleDateString()} at ${profileDate.toLocaleTimeString()}`, rowCount: profile.rowCount?.toString() || 'unknown', columnCount: profile.columnCount?.toString() || 'unknown', + sizeInBytes: profile.sizeInBytes ? bytesFormatter(profile.sizeInBytes) : 'unknown', }; }); @@ -59,6 +67,11 @@ export default function ProfilingRunsChart({ profiles }: Props) { key: 'Column Count', dataIndex: 'columnCount', }, + { + title: 'Size', + key: 'Size', + dataIndex: 'sizeInBytes', + }, ]; const selectedProfile = (selectedProfileIndex >= 0 && profiles[selectedProfileIndex]) || undefined; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/historical/charts/StatChart.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/historical/charts/StatChart.tsx index a0531f33d1b3a..db5b1a59759b1 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/historical/charts/StatChart.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/historical/charts/StatChart.tsx @@ -14,8 +14,9 @@ const ChartTitle = styled(Typography.Text)` } `; -const ChartCard = styled(Card)` +const ChartCard = styled(Card)<{ visible: boolean }>` box-shadow: ${(props) => props.theme.styles['box-shadow']}; + visibility: ${(props) => (props.visible ? 'visible' : 'hidden')}; ; `; type Point = { @@ -23,11 +24,17 @@ type Point = { value: number; }; +type AxisConfig = { + formatter: (tick: number) => string; +}; + export type Props = { title: string; values: Array; tickInterval: DateInterval; dateRange: DateRange; + yAxis?: AxisConfig; + visible?: boolean; }; /** @@ -41,7 +48,7 @@ const DEFAULT_AXIS_WIDTH = 2; /** * Time Series Chart with a single line. */ -export default function StatChart({ title, values, tickInterval: interval, dateRange }: Props) { +export default function StatChart({ title, values, tickInterval: interval, dateRange, yAxis, visible = true }: Props) { const timeSeriesData = useMemo( () => values @@ -56,22 +63,10 @@ export default function StatChart({ title, values, tickInterval: interval, dateR [values], ); - const chartData = { - title, - lines: [ - { - name: 'line_1', - data: timeSeriesData, - }, - ], - interval, - dateRange, - }; - return ( - + - {chartData.title} + {title} From 217151ea55c1b50712fa3f9287bdd469894aef7b Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Thu, 6 Jul 2023 14:15:58 -0500 Subject: [PATCH 102/222] =?UTF-8?q?fix(schema-registry):=20fix=20internal?= =?UTF-8?q?=20schema=20reg=20with=20custom=20duhe=20topic=20=E2=80=A6=20(#?= =?UTF-8?q?8371)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DatahubUpgradeNoSchemaRegistryTest.java | 3 ++- .../DUHESchemaRegistryFactory.java | 10 +++++++++ .../kafka/DataHubUpgradeKafkaListener.java | 2 +- .../boot/kafka/MockDUHEDeserializer.java | 16 +++++++++++--- .../boot/kafka/MockDUHESerializer.java | 21 ++++++++++++++++--- 5 files changed, 44 insertions(+), 8 deletions(-) diff --git a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNoSchemaRegistryTest.java b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNoSchemaRegistryTest.java index dd47e049dd187..db697a40d0c6c 100644 --- a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNoSchemaRegistryTest.java +++ b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNoSchemaRegistryTest.java @@ -19,7 +19,8 @@ @ActiveProfiles("test") @SpringBootTest(classes = {UpgradeCliApplication.class, UpgradeCliApplicationTestConfiguration.class}, properties = { - "kafka.schemaRegistry.type=INTERNAL" + "kafka.schemaRegistry.type=INTERNAL", + "DATAHUB_UPGRADE_HISTORY_TOPIC_NAME=test_due_topic" }) public class DatahubUpgradeNoSchemaRegistryTest extends AbstractTestNGSpringContextTests { diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/DUHESchemaRegistryFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/DUHESchemaRegistryFactory.java index 440d7a3cf2ab8..e282cc3861b41 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/DUHESchemaRegistryFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/DUHESchemaRegistryFactory.java @@ -6,15 +6,24 @@ import com.linkedin.metadata.config.kafka.KafkaConfiguration; import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import java.util.HashMap; import java.util.Map; +import static com.linkedin.metadata.boot.kafka.DataHubUpgradeKafkaListener.TOPIC_NAME; + @Slf4j @Configuration public class DUHESchemaRegistryFactory { + + public static final String DUHE_SCHEMA_REGISTRY_TOPIC_KEY = "duheTopicName"; + + @Value(TOPIC_NAME) + private String duheTopicName; + /** * Configure Kafka Producer/Consumer processes with a custom schema registry. */ @@ -25,6 +34,7 @@ protected SchemaRegistryConfig duheSchemaRegistryConfig(ConfigurationProvider pr props.put(AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, kafkaConfiguration .getSchemaRegistry().getUrl()); + props.put(DUHE_SCHEMA_REGISTRY_TOPIC_KEY, duheTopicName); log.info("DataHub System Update Registry"); return new SchemaRegistryConfig(MockDUHESerializer.class, MockDUHEDeserializer.class, props); diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/DataHubUpgradeKafkaListener.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/DataHubUpgradeKafkaListener.java index 20889e6497a92..e3f3a9c5b76a5 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/DataHubUpgradeKafkaListener.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/DataHubUpgradeKafkaListener.java @@ -38,7 +38,7 @@ public class DataHubUpgradeKafkaListener implements ConsumerSeekAware, Bootstrap private static final String CONSUMER_GROUP = "${DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID:generic-duhe-consumer-job-client}"; private static final String SUFFIX = "temp"; - private static final String TOPIC_NAME = "${DATAHUB_UPGRADE_HISTORY_TOPIC_NAME:" + Topics.DATAHUB_UPGRADE_HISTORY_TOPIC_NAME + "}"; + public static final String TOPIC_NAME = "${DATAHUB_UPGRADE_HISTORY_TOPIC_NAME:" + Topics.DATAHUB_UPGRADE_HISTORY_TOPIC_NAME + "}"; private final DefaultKafkaConsumerFactory _defaultKafkaConsumerFactory; diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/MockDUHEDeserializer.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/MockDUHEDeserializer.java index eef68b427e4de..408b212d52f48 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/MockDUHEDeserializer.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/MockDUHEDeserializer.java @@ -12,7 +12,8 @@ import java.io.IOException; import java.util.Map; -import static com.linkedin.metadata.boot.kafka.MockDUHESerializer.DATAHUB_UPGRADE_HISTORY_EVENT_SUBJECT; +import static com.linkedin.gms.factory.kafka.schemaregistry.DUHESchemaRegistryFactory.DUHE_SCHEMA_REGISTRY_TOPIC_KEY; +import static com.linkedin.metadata.boot.kafka.MockDUHESerializer.topicToSubjectName; /** * Used for early bootstrap to avoid contact with not yet existing schema registry @@ -20,11 +21,14 @@ @Slf4j public class MockDUHEDeserializer extends KafkaAvroDeserializer { + private String topicName; + public MockDUHEDeserializer() { this.schemaRegistry = buildMockSchemaRegistryClient(); } public MockDUHEDeserializer(SchemaRegistryClient client) { + super(client); this.schemaRegistry = buildMockSchemaRegistryClient(); } @@ -33,10 +37,16 @@ public MockDUHEDeserializer(SchemaRegistryClient client, Map props) { this.schemaRegistry = buildMockSchemaRegistryClient(); } - private static MockSchemaRegistryClient buildMockSchemaRegistryClient() { + @Override + public void configure(Map configs, boolean isKey) { + super.configure(configs, isKey); + topicName = configs.get(DUHE_SCHEMA_REGISTRY_TOPIC_KEY).toString(); + } + + private MockSchemaRegistryClient buildMockSchemaRegistryClient() { MockSchemaRegistryClient schemaRegistry = new MockSchemaRegistryClient2(); try { - schemaRegistry.register(DATAHUB_UPGRADE_HISTORY_EVENT_SUBJECT, + schemaRegistry.register(topicToSubjectName(topicName), new AvroSchema(EventUtils.ORIGINAL_DUHE_AVRO_SCHEMA)); return schemaRegistry; } catch (IOException | RestClientException e) { diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/MockDUHESerializer.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/MockDUHESerializer.java index 2ce66243ddb4e..1421f952289b3 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/MockDUHESerializer.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/MockDUHESerializer.java @@ -11,19 +11,24 @@ import java.io.IOException; import java.util.Map; +import static com.linkedin.gms.factory.kafka.schemaregistry.DUHESchemaRegistryFactory.DUHE_SCHEMA_REGISTRY_TOPIC_KEY; + /** * Used for early bootstrap to avoid contact with not yet existing schema registry */ @Slf4j public class MockDUHESerializer extends KafkaAvroSerializer { - static final String DATAHUB_UPGRADE_HISTORY_EVENT_SUBJECT = "DataHubUpgradeHistory_v1-value"; + private static final String DATAHUB_UPGRADE_HISTORY_EVENT_SUBJECT_SUFFIX = "-value"; + + private String topicName; public MockDUHESerializer() { this.schemaRegistry = buildMockSchemaRegistryClient(); } public MockDUHESerializer(SchemaRegistryClient client) { + super(client); this.schemaRegistry = buildMockSchemaRegistryClient(); } @@ -32,14 +37,24 @@ public MockDUHESerializer(SchemaRegistryClient client, Map props) { this.schemaRegistry = buildMockSchemaRegistryClient(); } - private static MockSchemaRegistryClient buildMockSchemaRegistryClient() { + @Override + public void configure(Map configs, boolean isKey) { + super.configure(configs, isKey); + topicName = configs.get(DUHE_SCHEMA_REGISTRY_TOPIC_KEY).toString(); + } + + private MockSchemaRegistryClient buildMockSchemaRegistryClient() { MockSchemaRegistryClient schemaRegistry = new MockSchemaRegistryClient(); try { - schemaRegistry.register(DATAHUB_UPGRADE_HISTORY_EVENT_SUBJECT, + schemaRegistry.register(topicToSubjectName(topicName), new AvroSchema(EventUtils.ORIGINAL_DUHE_AVRO_SCHEMA)); return schemaRegistry; } catch (IOException | RestClientException e) { throw new RuntimeException(e); } } + + public static String topicToSubjectName(String topicName) { + return topicName + DATAHUB_UPGRADE_HISTORY_EVENT_SUBJECT_SUFFIX; + } } From 5d3c6a8a002153c076883816cb9939db484ba9d7 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Thu, 6 Jul 2023 15:41:38 -0400 Subject: [PATCH 103/222] fix(java) Add try catch block when backfilling browse v2 (#8377) --- .../metadata/boot/steps/BackfillBrowsePathsV2Step.java | 9 +++++++-- .../boot/steps/BackfillBrowsePathsV2StepTest.java | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java index afe82f183b5de..627c5934140a8 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java @@ -40,7 +40,7 @@ public class BackfillBrowsePathsV2Step extends UpgradeStep { Constants.ML_FEATURE_TABLE_ENTITY_NAME, Constants.ML_FEATURE_ENTITY_NAME ); - private static final String VERSION = "1"; + private static final String VERSION = "2"; private static final String UPGRADE_ID = "backfill-default-browse-paths-v2-step"; private static final Integer BATCH_SIZE = 5000; @@ -114,7 +114,12 @@ private String backfillBrowsePathsV2(String entityType, AuditStamp auditStamp, S } for (SearchEntity searchEntity : scrollResult.getEntities()) { - ingestBrowsePathsV2(searchEntity.getEntity(), auditStamp); + try { + ingestBrowsePathsV2(searchEntity.getEntity(), auditStamp); + } catch (Exception e) { + // don't stop the whole step because of one bad urn or one bad ingestion + log.error(String.format("Error ingesting default browsePathsV2 aspect for urn %s", searchEntity.getEntity()), e); + } } return scrollResult.getScrollId(); diff --git a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2StepTest.java b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2StepTest.java index f02b07f7a5fcb..49fce75ab7c61 100644 --- a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2StepTest.java +++ b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2StepTest.java @@ -32,7 +32,7 @@ import static com.linkedin.metadata.Constants.CONTAINER_ASPECT_NAME; public class BackfillBrowsePathsV2StepTest { - private static final String VERSION_1 = "1"; + private static final String VERSION = "2"; private static final String UPGRADE_URN = String.format( "urn:li:%s:%s", Constants.DATA_HUB_UPGRADE_ENTITY_NAME, @@ -110,7 +110,7 @@ public void testDoesNotRunWhenAlreadyExecuted() throws Exception { final SearchService mockSearchService = initMockSearchService(); final Urn upgradeEntityUrn = Urn.createFromString(UPGRADE_URN); - com.linkedin.upgrade.DataHubUpgradeRequest upgradeRequest = new com.linkedin.upgrade.DataHubUpgradeRequest().setVersion(VERSION_1); + com.linkedin.upgrade.DataHubUpgradeRequest upgradeRequest = new com.linkedin.upgrade.DataHubUpgradeRequest().setVersion(VERSION); Map upgradeRequestAspects = new HashMap<>(); upgradeRequestAspects.put(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(upgradeRequest.data()))); From 294857ab8d0651db5413d4aae90198b8be87597c Mon Sep 17 00:00:00 2001 From: Dmitry Bryazgin <58312247+bda618@users.noreply.github.com> Date: Thu, 6 Jul 2023 16:15:16 -0400 Subject: [PATCH 104/222] feat(ingest): Add advanced configuration for LDAP manager ingestion (#7784) Co-authored-by: Harshal Sheth --- .../src/datahub/ingestion/source/ldap.py | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/ldap.py b/metadata-ingestion/src/datahub/ingestion/source/ldap.py index ef35526792d6c..f2a9e95a1db2c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ldap.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ldap.py @@ -128,6 +128,16 @@ class LDAPSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin): default=20, description="Size of each page to fetch when extracting metadata." ) + manager_filter_enabled: bool = Field( + default=True, + description="Use LDAP extractor filter to search managers.", + ) + + manager_pagination_enabled: bool = Field( + default=True, + description="Use pagination while search for managers (enabled by default).", + ) + # default mapping for attrs user_attrs_map: Dict[str, Any] = {} group_attrs_map: Dict[str, Any] = {} @@ -286,11 +296,19 @@ def handle_user(self, dn: str, attrs: Dict[str, Any]) -> Iterable[MetadataWorkUn if self.config.user_attrs_map["managerUrn"] in attrs: try: m_cn = attrs[self.config.user_attrs_map["managerUrn"]][0].decode() + if self.config.manager_filter_enabled: + manager_filter = self.config.filter + else: + manager_filter = None + if self.config.manager_pagination_enabled: + ctrls = [self.lc] + else: + ctrls = None manager_msgid = self.ldap_client.search_ext( m_cn, ldap.SCOPE_BASE, - self.config.filter, - serverctrls=[self.lc], + manager_filter, + serverctrls=ctrls, ) result = self.ldap_client.result3(manager_msgid) if result[1]: From 8cf778dc9b2bb77f9af1ffd979e65fb283579938 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Fri, 7 Jul 2023 01:46:07 +0530 Subject: [PATCH 105/222] fix(ingest): update pydantic helpers to address unique name issue (#8324) Co-authored-by: Aseem Bansal --- .../pydantic_field_deprecation.py | 4 ++ .../configuration/validate_field_removal.py | 8 +++- .../tests/unit/test_pydantic_validators.py | 41 ++++++++++++++++++- 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/metadata-ingestion/src/datahub/configuration/pydantic_field_deprecation.py b/metadata-ingestion/src/datahub/configuration/pydantic_field_deprecation.py index 2cb9ea5401117..ed82acb594ed7 100644 --- a/metadata-ingestion/src/datahub/configuration/pydantic_field_deprecation.py +++ b/metadata-ingestion/src/datahub/configuration/pydantic_field_deprecation.py @@ -19,4 +19,8 @@ def _validate_deprecated(cls: Type, values: dict) -> dict: warnings.warn(output, ConfigurationWarning, stacklevel=2) return values + # Hack: Pydantic maintains unique list of validators by referring its __name__. + # https://github.com/pydantic/pydantic/blob/v1.10.9/pydantic/main.py#L264 + # This hack ensures that multiple field deprecated do not overwrite each other. + _validate_deprecated.__name__ = f"{_validate_deprecated.__name__}_{field}" return pydantic.root_validator(pre=True, allow_reuse=True)(_validate_deprecated) diff --git a/metadata-ingestion/src/datahub/configuration/validate_field_removal.py b/metadata-ingestion/src/datahub/configuration/validate_field_removal.py index 8f60953760808..6b8f0df0588e6 100644 --- a/metadata-ingestion/src/datahub/configuration/validate_field_removal.py +++ b/metadata-ingestion/src/datahub/configuration/validate_field_removal.py @@ -10,7 +10,7 @@ def pydantic_removed_field( field: str, print_warning: bool = True, ) -> classmethod: - def _validate_field_rename(cls: Type, values: dict) -> dict: + def _validate_field_removal(cls: Type, values: dict) -> dict: if field in values: if print_warning: warnings.warn( @@ -21,4 +21,8 @@ def _validate_field_rename(cls: Type, values: dict) -> dict: values.pop(field) return values - return pydantic.root_validator(pre=True, allow_reuse=True)(_validate_field_rename) + # Hack: Pydantic maintains unique list of validators by referring its __name__. + # https://github.com/pydantic/pydantic/blob/v1.10.9/pydantic/main.py#L264 + # This hack ensures that multiple field removals do not overwrite each other. + _validate_field_removal.__name__ = f"{_validate_field_removal.__name__}_{field}" + return pydantic.root_validator(pre=True, allow_reuse=True)(_validate_field_removal) diff --git a/metadata-ingestion/tests/unit/test_pydantic_validators.py b/metadata-ingestion/tests/unit/test_pydantic_validators.py index 8eaf4e5f4548b..07d86043a35bf 100644 --- a/metadata-ingestion/tests/unit/test_pydantic_validators.py +++ b/metadata-ingestion/tests/unit/test_pydantic_validators.py @@ -1,15 +1,20 @@ +from typing import Optional + import pytest from pydantic import ValidationError from datahub.configuration.common import ConfigModel +from datahub.configuration.pydantic_field_deprecation import pydantic_field_deprecated +from datahub.configuration.validate_field_removal import pydantic_removed_field from datahub.configuration.validate_field_rename import pydantic_renamed_field +from datahub.utilities.global_warning_util import get_global_warnings def test_field_rename(): class TestModel(ConfigModel): b: str - _validate_deprecated = pydantic_renamed_field("a", "b") + _validate_rename = pydantic_renamed_field("a", "b") v = TestModel.parse_obj({"b": "original"}) assert v.b == "original" @@ -54,3 +59,37 @@ class TestModel(ConfigModel): with pytest.raises(ValidationError): TestModel.parse_obj({}) + + +def test_field_remove(): + class TestModel(ConfigModel): + b: str + + _validate_removed_r1 = pydantic_removed_field("r1") + _validate_removed_r2 = pydantic_removed_field("r2") + + v = TestModel.parse_obj({"b": "original"}) + assert v.b == "original" + + v = TestModel.parse_obj({"b": "original", "r1": "removed", "r2": "removed"}) + assert v.b == "original" + + +def test_field_deprecated(): + class TestModel(ConfigModel): + d1: Optional[str] + d2: Optional[str] + b: str + + _validate_deprecated_d1 = pydantic_field_deprecated("d1") + _validate_deprecated_d2 = pydantic_field_deprecated("d2") + + v = TestModel.parse_obj({"b": "original"}) + assert v.b == "original" + + v = TestModel.parse_obj({"b": "original", "d1": "deprecated", "d2": "deprecated"}) + assert v.b == "original" + assert v.d1 == "deprecated" + assert v.d2 == "deprecated" + assert any(["d1 is deprecated" in warning for warning in get_global_warnings()]) + assert any(["d2 is deprecated" in warning for warning in get_global_warnings()]) From b8f5074763f754d6c8c552adbd075a53e2a6fc89 Mon Sep 17 00:00:00 2001 From: Segun Sogunle <102164363+segun-s@users.noreply.github.com> Date: Thu, 6 Jul 2023 21:16:46 +0100 Subject: [PATCH 106/222] fix(cli): local variable reference before assignment (#8222) Co-authored-by: John Joyce --- metadata-ingestion/src/datahub/cli/migrate.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metadata-ingestion/src/datahub/cli/migrate.py b/metadata-ingestion/src/datahub/cli/migrate.py index f40a6194806d1..7a079b228ffed 100644 --- a/metadata-ingestion/src/datahub/cli/migrate.py +++ b/metadata-ingestion/src/datahub/cli/migrate.py @@ -141,6 +141,9 @@ def dataplatform2instance_func( migration_report = MigrationReport(run_id, dry_run, keep) system_metadata = SystemMetadataClass(runId=run_id) + # initialize for dry-run + graph = DataHubGraph(config=DataHubGraphConfig(server="127.0.0.1")) + if not dry_run: graph = DataHubGraph( config=DataHubGraphConfig(server=cli_utils.get_session_and_host()[1]) From 3a21c27f06998787964595919a692b9da5916750 Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Thu, 6 Jul 2023 16:43:42 -0400 Subject: [PATCH 107/222] feat(ingest): Turn on browse path v2 creation (#8342) Co-authored-by: Harshal Sheth --- .../datahub/ingestion/run/pipeline_config.py | 2 +- .../src/datahub/ingestion/source/tableau.py | 2 +- .../bigquery_v2/bigquery_mcp_golden.json | 59 + .../clickhouse/clickhouse_mces_golden.json | 241 ++- .../hive/hive_mces_all_db_golden.json | 237 ++- .../integration/hive/hive_mces_golden.json | 183 +- .../integration/kafka/kafka_mces_golden.json | 185 +- .../looker/golden_looker_mces.json | 244 ++- .../looker/golden_test_allow_ingest.json | 159 +- ...olden_test_external_project_view_mces.json | 22 + .../golden_test_independent_look_ingest.json | 44 + .../looker/golden_test_ingest.json | 159 +- .../looker/golden_test_ingest_joins.json | 159 +- .../golden_test_ingest_unaliased_joins.json | 159 +- .../looker_mces_golden_deleted_stateful.json | 182 +- .../looker/looker_mces_usage_history.json | 257 ++- .../integration/lookml/expected_output.json | 325 +++ .../lookml/lookml_mces_api_bigquery.json | 325 +++ .../lookml/lookml_mces_api_hive2.json | 325 +++ .../lookml/lookml_mces_badsql_parser.json | 325 +++ .../lookml_mces_golden_deleted_stateful.json | 72 +- .../lookml/lookml_mces_offline.json | 325 +++ .../lookml_mces_offline_deny_pattern.json | 125 ++ ...lookml_mces_offline_platform_instance.json | 325 +++ .../lookml_mces_with_external_urls.json | 325 +++ .../lookml/lookml_reachable_views.json | 75 + .../refinement_include_order_golden.json | 100 + .../lookml/refinements_ingestion_golden.json | 325 +++ .../mysql/mysql_mces_no_db_golden.json | 240 +++ .../mysql/mysql_mces_with_db_golden.json | 55 + .../golden_test_ingest_with_database.json | 151 ++ .../golden_test_ingest_with_out_database.json | 151 ++ .../postgres_all_db_mces_with_db_golden.json | 94 + .../postgres_mces_with_db_golden.json | 59 + .../golden_test_admin_access_not_allowed.json | 57 + .../powerbi/golden_test_admin_only.json | 76 + .../powerbi/golden_test_container.json | 315 +++ .../golden_test_disabled_ownership.json | 57 + .../powerbi/golden_test_endorsement.json | 57 + .../powerbi/golden_test_ingest.json | 57 + .../powerbi/golden_test_lineage.json | 57 + .../golden_test_lower_case_urn_ingest.json | 57 + ..._config_and_modified_since_admin_only.json | 175 ++ .../golden_test_platform_instance_ingest.json | 57 + .../powerbi/golden_test_report.json | 114 ++ .../golden_test_scan_all_workspaces.json | 76 + ...lden_test_server_to_platform_instance.json | 57 + .../presto_on_hive_mces_golden_1.json | 265 ++- .../presto_on_hive_mces_golden_2.json | 241 ++- .../presto_on_hive_mces_golden_3.json | 265 ++- .../presto_on_hive_mces_golden_4.json | 241 ++- .../golden_mces_file_without_extension.json | 341 +++- .../golden_mces_folder_no_partition.json | 339 +++- ...lden_mces_folder_no_partition_exclude.json | 289 ++- ...den_mces_folder_no_partition_filename.json | 339 +++- .../golden_mces_folder_no_partition_glob.json | 289 ++- .../golden_mces_folder_partition_basic.json | 289 ++- .../golden_mces_folder_partition_keyval.json | 289 ++- ...n_mces_folder_partition_update_schema.json | 289 ++- .../local/golden_mces_multiple_files.json | 589 +++++- .../golden_mces_multiple_spec_for_files.json | 339 +++- ...s_multiple_specs_of_different_buckets.json | 339 +++- .../local/golden_mces_single_file.json | 289 ++- .../golden_mces_file_without_extension.json | 157 +- .../s3/golden_mces_folder_no_partition.json | 155 +- ...lden_mces_folder_no_partition_exclude.json | 121 +- ...den_mces_folder_no_partition_filename.json | 155 +- .../golden_mces_folder_no_partition_glob.json | 121 +- .../golden_mces_folder_partition_basic.json | 121 +- .../golden_mces_folder_partition_keyval.json | 121 +- ...n_mces_folder_partition_update_schema.json | 121 +- .../s3/golden_mces_multiple_files.json | 325 ++- .../golden_mces_multiple_spec_for_files.json | 155 +- ...s_multiple_specs_of_different_buckets.json | 242 ++- .../s3/golden_mces_single_file.json | 121 +- .../snowflake/snowflake_golden.json | 347 +++- .../snowflake_privatelink_golden.json | 275 +++ .../golden_mces_mssql_no_db_to_file.json | 780 ++++++- .../golden_mces_mssql_no_db_with_filter.json | 391 ++++ .../golden_mces_mssql_to_file.json | 391 ++++ ...bleau_extract_all_project_mces_golden.json | 1630 ++++++++++++++- .../tableau/tableau_mces_golden.json | 1586 ++++++++++++++- .../tableau_mces_golden_deleted_stateful.json | 299 +-- .../tableau_nested_project_mces_golden.json | 1630 ++++++++++++++- .../tableau_project_path_mces_golden.json | 59 + .../tableau_signout_timeout_mces_golden.json | 1586 ++++++++++++++- ...au_with_platform_instance_mces_golden.json | 1792 +++++++++++++++-- .../trino/trino_hive_mces_golden.json | 269 ++- .../integration/trino/trino_mces_golden.json | 131 ++ .../unity/unity_catalog_mces_golden.json | 543 +++++ .../glue/glue_deleted_actor_mces_golden.json | 63 +- 91 files changed, 25297 insertions(+), 650 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py b/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py index 83b73d144ec52..ff9a7a6f3d146 100644 --- a/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py +++ b/metadata-ingestion/src/datahub/ingestion/run/pipeline_config.py @@ -50,7 +50,7 @@ class FlagsConfig(ConfigModel): ) generate_browse_path_v2_dry_run: bool = Field( - default=True, + default=False, description=( "Run through browse paths v2 generation but do not actually write the aspects to DataHub. " "Requires `generate_browse_path_v2` to also be enabled." diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau.py index b1ba8145796d4..92f61870aa968 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau.py @@ -1689,7 +1689,7 @@ def emit_table( browse_paths = BrowsePathsClass( paths=[ f"/{self.config.env.lower()}/{self.platform}/{path}" - for path in database_table.paths + for path in sorted(database_table.paths, key=lambda p: (len(p), p)) ] ) dataset_snapshot.aspects.append(browse_paths) diff --git a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json index 38d36871fbde8..f58eee09aa1ce 100644 --- a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json +++ b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json @@ -66,6 +66,21 @@ "runId": "bigquery-2022_02_03-07_00_00" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:068bd9323110994a40019fcf6cfc60d3", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:8df46c5e3ded05a3122b0015822c0ef0", @@ -150,6 +165,26 @@ "runId": "bigquery-2022_02_03-07_00_00" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:8df46c5e3ded05a3122b0015822c0ef0", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:068bd9323110994a40019fcf6cfc60d3", + "urn": "urn:li:container:068bd9323110994a40019fcf6cfc60d3" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)", @@ -263,5 +298,29 @@ "lastObserved": 1643871600000, "runId": "bigquery-2022_02_03-07_00_00" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:068bd9323110994a40019fcf6cfc60d3", + "urn": "urn:li:container:068bd9323110994a40019fcf6cfc60d3" + }, + { + "id": "urn:li:container:8df46c5e3ded05a3122b0015822c0ef0", + "urn": "urn:li:container:8df46c5e3ded05a3122b0015822c0ef0" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "bigquery-2022_02_03-07_00_00" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/clickhouse/clickhouse_mces_golden.json b/metadata-ingestion/tests/integration/clickhouse/clickhouse_mces_golden.json index c44b512baadcd..67915ca6be713 100644 --- a/metadata-ingestion/tests/integration/clickhouse/clickhouse_mces_golden.json +++ b/metadata-ingestion/tests/integration/clickhouse/clickhouse_mces_golden.json @@ -68,6 +68,21 @@ "runId": "clickhouse-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "clickhouse-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:1154d1da73a95376c9f33f47694cf1de", @@ -153,6 +168,26 @@ "runId": "clickhouse-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1154d1da73a95376c9f33f47694cf1de", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c", + "urn": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "clickhouse-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:clickhouse,clickhousetestserver.db1.mv_target_table,PROD)", @@ -211,11 +246,11 @@ "primary_key": "col_Int64", "sampling_key": "", "storage_policy": "default", - "metadata_modification_time": "2023-05-11 23:34:18", + "metadata_modification_time": "2023-07-03 18:52:30", "total_rows": "10", "total_bytes": "671", - "data_paths": "['/var/lib/clickhouse/store/689/689251e9-d514-4a89-ac7b-14857a8a395a/']", - "metadata_path": "/var/lib/clickhouse/store/bf5/bf553974-954c-4cca-8a8f-c56557cdcfe0/mv_target_table.sql" + "data_paths": "['/var/lib/clickhouse/store/2d4/2d43771f-2a9f-4a28-962e-992a7c08102f/']", + "metadata_path": "/var/lib/clickhouse/store/e2e/e2e3221c-2ffa-4009-b71e-ad306b778310/mv_target_table.sql" }, "name": "mv_target_table", "description": "This is target table for materialized view", @@ -351,6 +386,30 @@ "runId": "clickhouse-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:clickhouse,clickhousetestserver.db1.mv_target_table,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c", + "urn": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c" + }, + { + "id": "urn:li:container:1154d1da73a95376c9f33f47694cf1de", + "urn": "urn:li:container:1154d1da73a95376c9f33f47694cf1de" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "clickhouse-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:clickhouse,clickhousetestserver.db1.test_data_types,PROD)", @@ -385,11 +444,11 @@ "primary_key": "", "sampling_key": "", "storage_policy": "default", - "metadata_modification_time": "2023-05-11 23:34:18", + "metadata_modification_time": "2023-07-03 18:52:30", "total_rows": "0", "total_bytes": "0", - "data_paths": "['/var/lib/clickhouse/store/1bb/1bb7aa71-5c5e-426b-ba29-a12b800e6dcf/']", - "metadata_path": "/var/lib/clickhouse/store/bf5/bf553974-954c-4cca-8a8f-c56557cdcfe0/test_data_types.sql" + "data_paths": "['/var/lib/clickhouse/store/700/70013972-f4ad-4c8b-a4bd-b397c9cc727f/']", + "metadata_path": "/var/lib/clickhouse/store/e2e/e2e3221c-2ffa-4009-b71e-ad306b778310/test_data_types.sql" }, "name": "test_data_types", "description": "This table has basic types", @@ -954,6 +1013,30 @@ "runId": "clickhouse-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:clickhouse,clickhousetestserver.db1.test_data_types,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c", + "urn": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c" + }, + { + "id": "urn:li:container:1154d1da73a95376c9f33f47694cf1de", + "urn": "urn:li:container:1154d1da73a95376c9f33f47694cf1de" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "clickhouse-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:clickhouse,clickhousetestserver.db1.test_dict,PROD)", @@ -1012,11 +1095,11 @@ "primary_key": "", "sampling_key": "", "storage_policy": "", - "metadata_modification_time": "2023-05-11 23:34:18", + "metadata_modification_time": "2023-07-03 18:52:30", "total_rows": "None", "total_bytes": "None", "data_paths": "[]", - "metadata_path": "/var/lib/clickhouse/store/bf5/bf553974-954c-4cca-8a8f-c56557cdcfe0/test_dict.sql" + "metadata_path": "/var/lib/clickhouse/store/e2e/e2e3221c-2ffa-4009-b71e-ad306b778310/test_dict.sql" }, "name": "test_dict", "description": "", @@ -1113,6 +1196,30 @@ "runId": "clickhouse-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:clickhouse,clickhousetestserver.db1.test_dict,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c", + "urn": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c" + }, + { + "id": "urn:li:container:1154d1da73a95376c9f33f47694cf1de", + "urn": "urn:li:container:1154d1da73a95376c9f33f47694cf1de" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "clickhouse-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:clickhouse,clickhousetestserver.db1.test_nested_data_types,PROD)", @@ -1147,11 +1254,11 @@ "primary_key": "", "sampling_key": "", "storage_policy": "default", - "metadata_modification_time": "2023-05-11 23:34:18", + "metadata_modification_time": "2023-07-03 18:52:30", "total_rows": "0", "total_bytes": "0", - "data_paths": "['/var/lib/clickhouse/store/531/531fc424-a69b-4d7c-8b4c-a7dbbc2e919f/']", - "metadata_path": "/var/lib/clickhouse/store/bf5/bf553974-954c-4cca-8a8f-c56557cdcfe0/test_nested_data_types.sql" + "data_paths": "['/var/lib/clickhouse/store/04c/04c9735e-af23-4850-a034-8da8f03d75af/']", + "metadata_path": "/var/lib/clickhouse/store/e2e/e2e3221c-2ffa-4009-b71e-ad306b778310/test_nested_data_types.sql" }, "name": "test_nested_data_types", "description": "This table has nested types", @@ -1352,6 +1459,30 @@ "runId": "clickhouse-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:clickhouse,clickhousetestserver.db1.test_nested_data_types,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c", + "urn": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c" + }, + { + "id": "urn:li:container:1154d1da73a95376c9f33f47694cf1de", + "urn": "urn:li:container:1154d1da73a95376c9f33f47694cf1de" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "clickhouse-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:clickhouse,clickhousetestserver.db1.mv_with_target_table,PROD)", @@ -1410,11 +1541,11 @@ "primary_key": "", "sampling_key": "", "storage_policy": "", - "metadata_modification_time": "2023-05-11 23:34:18", + "metadata_modification_time": "2023-07-03 18:52:30", "total_rows": "None", "total_bytes": "None", - "data_paths": "['/var/lib/clickhouse/store/689/689251e9-d514-4a89-ac7b-14857a8a395a/']", - "metadata_path": "/var/lib/clickhouse/store/bf5/bf553974-954c-4cca-8a8f-c56557cdcfe0/mv_with_target_table.sql", + "data_paths": "['/var/lib/clickhouse/store/2d4/2d43771f-2a9f-4a28-962e-992a7c08102f/']", + "metadata_path": "/var/lib/clickhouse/store/e2e/e2e3221c-2ffa-4009-b71e-ad306b778310/mv_with_target_table.sql", "view_definition": "", "is_view": "True" }, @@ -1569,6 +1700,30 @@ "runId": "clickhouse-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:clickhouse,clickhousetestserver.db1.mv_with_target_table,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c", + "urn": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c" + }, + { + "id": "urn:li:container:1154d1da73a95376c9f33f47694cf1de", + "urn": "urn:li:container:1154d1da73a95376c9f33f47694cf1de" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "clickhouse-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:clickhouse,clickhousetestserver.db1.mv_without_target_table,PROD)", @@ -1627,11 +1782,11 @@ "primary_key": "", "sampling_key": "", "storage_policy": "", - "metadata_modification_time": "2023-05-11 23:34:18", + "metadata_modification_time": "2023-07-03 18:52:30", "total_rows": "0", "total_bytes": "0", - "data_paths": "['/var/lib/clickhouse/store/42b/42b5020e-8007-4b11-b491-f4604a33d2e9/']", - "metadata_path": "/var/lib/clickhouse/store/bf5/bf553974-954c-4cca-8a8f-c56557cdcfe0/mv_without_target_table.sql", + "data_paths": "['/var/lib/clickhouse/store/649/64934dd8-0347-4e30-a604-8b27022dc799/']", + "metadata_path": "/var/lib/clickhouse/store/e2e/e2e3221c-2ffa-4009-b71e-ad306b778310/mv_without_target_table.sql", "view_definition": "", "is_view": "True" }, @@ -1786,6 +1941,30 @@ "runId": "clickhouse-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:clickhouse,clickhousetestserver.db1.mv_without_target_table,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c", + "urn": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c" + }, + { + "id": "urn:li:container:1154d1da73a95376c9f33f47694cf1de", + "urn": "urn:li:container:1154d1da73a95376c9f33f47694cf1de" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "clickhouse-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:clickhouse,clickhousetestserver.db1.test_view,PROD)", @@ -1844,11 +2023,11 @@ "primary_key": "", "sampling_key": "", "storage_policy": "", - "metadata_modification_time": "2023-05-11 23:34:18", + "metadata_modification_time": "2023-07-03 18:52:30", "total_rows": "None", "total_bytes": "None", "data_paths": "[]", - "metadata_path": "/var/lib/clickhouse/store/bf5/bf553974-954c-4cca-8a8f-c56557cdcfe0/test_view.sql", + "metadata_path": "/var/lib/clickhouse/store/e2e/e2e3221c-2ffa-4009-b71e-ad306b778310/test_view.sql", "view_definition": "", "is_view": "True" }, @@ -1950,5 +2129,29 @@ "lastObserved": 1586847600000, "runId": "clickhouse-test" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:clickhouse,clickhousetestserver.db1.test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c", + "urn": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c" + }, + { + "id": "urn:li:container:1154d1da73a95376c9f33f47694cf1de", + "urn": "urn:li:container:1154d1da73a95376c9f33f47694cf1de" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "clickhouse-test" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/hive/hive_mces_all_db_golden.json b/metadata-ingestion/tests/integration/hive/hive_mces_all_db_golden.json index 737b5e2f5dcc5..f3b6d2b8138cc 100644 --- a/metadata-ingestion/tests/integration/hive/hive_mces_all_db_golden.json +++ b/metadata-ingestion/tests/integration/hive/hive_mces_all_db_golden.json @@ -66,6 +66,21 @@ "runId": "hive-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ded36d15fcfbbb939830549697122661", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", @@ -96,7 +111,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu May 11 23:37:23 UTC 2023", + "CreateTime:": "Wed Jul 05 17:38:26 UTC 2023", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore", @@ -106,7 +121,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1683848243", + "Table Parameters: transient_lastDdlTime": "1688578706", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -192,6 +207,26 @@ "runId": "hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ded36d15fcfbbb939830549697122661", + "urn": "urn:li:container:ded36d15fcfbbb939830549697122661" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", @@ -222,7 +257,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu May 11 23:37:23 UTC 2023", + "CreateTime:": "Wed Jul 05 17:38:26 UTC 2023", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test", @@ -232,7 +267,7 @@ "Table Parameters: numRows": "1", "Table Parameters: rawDataSize": "32", "Table Parameters: totalSize": "33", - "Table Parameters: transient_lastDdlTime": "1683848246", + "Table Parameters: transient_lastDdlTime": "1688578710", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -353,6 +388,26 @@ "runId": "hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ded36d15fcfbbb939830549697122661", + "urn": "urn:li:container:ded36d15fcfbbb939830549697122661" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", @@ -383,7 +438,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu May 11 23:37:27 UTC 2023", + "CreateTime:": "Wed Jul 05 17:38:30 UTC 2023", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test", @@ -393,7 +448,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1683848247", + "Table Parameters: transient_lastDdlTime": "1688578710", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -483,6 +538,26 @@ "runId": "hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ded36d15fcfbbb939830549697122661", + "urn": "urn:li:container:ded36d15fcfbbb939830549697122661" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", @@ -513,7 +588,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu May 11 23:37:27 UTC 2023", + "CreateTime:": "Wed Jul 05 17:38:30 UTC 2023", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test", @@ -523,7 +598,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1683848247", + "Table Parameters: transient_lastDdlTime": "1688578710", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -662,6 +737,26 @@ "runId": "hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ded36d15fcfbbb939830549697122661", + "urn": "urn:li:container:ded36d15fcfbbb939830549697122661" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", @@ -692,7 +787,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu May 11 23:37:20 UTC 2023", + "CreateTime:": "Wed Jul 05 17:38:22 UTC 2023", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", @@ -701,7 +796,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "5812", - "Table Parameters: transient_lastDdlTime": "1683848241", + "Table Parameters: transient_lastDdlTime": "1688578704", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -787,6 +882,26 @@ "runId": "hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ded36d15fcfbbb939830549697122661", + "urn": "urn:li:container:ded36d15fcfbbb939830549697122661" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", @@ -817,7 +932,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu May 11 23:37:23 UTC 2023", + "CreateTime:": "Wed Jul 05 17:38:26 UTC 2023", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test", @@ -827,7 +942,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1683848243", + "Table Parameters: transient_lastDdlTime": "1688578706", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -944,6 +1059,26 @@ "runId": "hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ded36d15fcfbbb939830549697122661", + "urn": "urn:li:container:ded36d15fcfbbb939830549697122661" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", @@ -974,7 +1109,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu May 11 23:37:27 UTC 2023", + "CreateTime:": "Wed Jul 05 17:38:30 UTC 2023", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test", @@ -984,7 +1119,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1683848247", + "Table Parameters: transient_lastDdlTime": "1688578710", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -1170,6 +1305,26 @@ "runId": "hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ded36d15fcfbbb939830549697122661", + "urn": "urn:li:container:ded36d15fcfbbb939830549697122661" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:8cc876554899e33efe67c389aaf29c4b", @@ -1237,6 +1392,21 @@ "runId": "hive-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:8cc876554899e33efe67c389aaf29c4b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db2.pokes,PROD)", @@ -1267,7 +1437,7 @@ "customProperties": { "Database:": "db2", "Owner:": "root", - "CreateTime:": "Thu May 11 23:37:22 UTC 2023", + "CreateTime:": "Wed Jul 05 17:38:24 UTC 2023", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db2.db/pokes", @@ -1276,7 +1446,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "5812", - "Table Parameters: transient_lastDdlTime": "1683848243", + "Table Parameters: transient_lastDdlTime": "1688578706", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -1365,6 +1535,26 @@ "runId": "hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db2.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:8cc876554899e33efe67c389aaf29c4b", + "urn": "urn:li:container:8cc876554899e33efe67c389aaf29c4b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:80a9564e8e33aacbf9a023fa43b8ee03", @@ -1431,5 +1621,20 @@ "lastObserved": 1586847600000, "runId": "hive-test" } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:80a9564e8e33aacbf9a023fa43b8ee03", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/hive/hive_mces_golden.json b/metadata-ingestion/tests/integration/hive/hive_mces_golden.json index 7a5d1f1bd786b..08f281f398909 100644 --- a/metadata-ingestion/tests/integration/hive/hive_mces_golden.json +++ b/metadata-ingestion/tests/integration/hive/hive_mces_golden.json @@ -66,6 +66,21 @@ "runId": "hive-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ded36d15fcfbbb939830549697122661", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", @@ -96,7 +111,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu May 11 23:37:23 UTC 2023", + "CreateTime:": "Wed Jul 05 17:38:26 UTC 2023", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore", @@ -106,7 +121,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1683848243", + "Table Parameters: transient_lastDdlTime": "1688578706", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -192,6 +207,26 @@ "runId": "hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ded36d15fcfbbb939830549697122661", + "urn": "urn:li:container:ded36d15fcfbbb939830549697122661" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", @@ -222,7 +257,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu May 11 23:37:23 UTC 2023", + "CreateTime:": "Wed Jul 05 17:38:26 UTC 2023", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test", @@ -232,7 +267,7 @@ "Table Parameters: numRows": "1", "Table Parameters: rawDataSize": "32", "Table Parameters: totalSize": "33", - "Table Parameters: transient_lastDdlTime": "1683848246", + "Table Parameters: transient_lastDdlTime": "1688578710", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -353,6 +388,26 @@ "runId": "hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ded36d15fcfbbb939830549697122661", + "urn": "urn:li:container:ded36d15fcfbbb939830549697122661" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", @@ -383,7 +438,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu May 11 23:37:27 UTC 2023", + "CreateTime:": "Wed Jul 05 17:38:30 UTC 2023", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test", @@ -393,7 +448,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1683848247", + "Table Parameters: transient_lastDdlTime": "1688578710", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -483,6 +538,26 @@ "runId": "hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ded36d15fcfbbb939830549697122661", + "urn": "urn:li:container:ded36d15fcfbbb939830549697122661" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", @@ -513,7 +588,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu May 11 23:37:27 UTC 2023", + "CreateTime:": "Wed Jul 05 17:38:30 UTC 2023", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test", @@ -523,7 +598,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1683848247", + "Table Parameters: transient_lastDdlTime": "1688578710", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -662,6 +737,26 @@ "runId": "hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ded36d15fcfbbb939830549697122661", + "urn": "urn:li:container:ded36d15fcfbbb939830549697122661" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", @@ -692,7 +787,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu May 11 23:37:20 UTC 2023", + "CreateTime:": "Wed Jul 05 17:38:22 UTC 2023", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", @@ -701,7 +796,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "5812", - "Table Parameters: transient_lastDdlTime": "1683848241", + "Table Parameters: transient_lastDdlTime": "1688578704", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -787,6 +882,26 @@ "runId": "hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ded36d15fcfbbb939830549697122661", + "urn": "urn:li:container:ded36d15fcfbbb939830549697122661" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", @@ -817,7 +932,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu May 11 23:37:23 UTC 2023", + "CreateTime:": "Wed Jul 05 17:38:26 UTC 2023", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test", @@ -827,7 +942,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1683848243", + "Table Parameters: transient_lastDdlTime": "1688578706", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -944,6 +1059,26 @@ "runId": "hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ded36d15fcfbbb939830549697122661", + "urn": "urn:li:container:ded36d15fcfbbb939830549697122661" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", @@ -974,7 +1109,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu May 11 23:37:27 UTC 2023", + "CreateTime:": "Wed Jul 05 17:38:30 UTC 2023", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test", @@ -984,7 +1119,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1683848247", + "Table Parameters: transient_lastDdlTime": "1688578710", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -1169,5 +1304,25 @@ "lastObserved": 1586847600000, "runId": "hive-test" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ded36d15fcfbbb939830549697122661", + "urn": "urn:li:container:ded36d15fcfbbb939830549697122661" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/kafka/kafka_mces_golden.json b/metadata-ingestion/tests/integration/kafka/kafka_mces_golden.json index 9f88ec78bcb9d..e51eaa10b8b10 100644 --- a/metadata-ingestion/tests/integration/kafka/kafka_mces_golden.json +++ b/metadata-ingestion/tests/integration/kafka/kafka_mces_golden.json @@ -2,7 +2,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,value_topic,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_topic,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -11,7 +11,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "value_topic", + "schemaName": "key_topic", "platform": "urn:li:dataPlatform:kafka", "version": 0, "created": { @@ -22,63 +22,38 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "hash": "dc1cf32c2688cc3d2d27fe6e856f06d2", + "hash": "44fd7a7b325d6fdd4275b1f02a79c1a8", "platformSchema": { "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"CreateUserRequest\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Value schema for kafka topic\",\"fields\":[{\"name\":\"email\",\"type\":\"string\"},{\"name\":\"firstName\",\"type\":\"string\"},{\"name\":\"lastName\",\"type\":\"string\"}]}", - "documentSchemaType": "AVRO", - "keySchema": "\"string\"", + "documentSchema": "", + "keySchema": "{\"type\":\"record\",\"name\":\"UserKey\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Key schema for kafka topic\",\"fields\":[{\"name\":\"id\",\"type\":\"long\"},{\"name\":\"namespace\",\"type\":\"string\"}]}", "keySchemaType": "AVRO" } }, "fields": [ { - "fieldPath": "[version=2.0].[key=True].[type=string]", + "fieldPath": "[version=2.0].[key=True].[type=UserKey].[type=long].id", "nullable": false, "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "string", + "nativeDataType": "id", "recursive": false, "isPartOfKey": true }, { - "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].email", - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "email", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].firstName", - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "firstName", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].lastName", + "fieldPath": "[version=2.0].[key=True].[type=UserKey].[type=string].namespace", "nullable": false, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "lastName", + "nativeDataType": "namespace", "recursive": false, - "isPartOfKey": false + "isPartOfKey": true } ] } @@ -102,8 +77,7 @@ "max.message.bytes": "1048588", "unclean.leader.election.enable": "false" }, - "name": "value_topic", - "description": "Value schema for kafka topic", + "name": "key_topic", "tags": [] } } @@ -117,7 +91,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,value_topic,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_topic,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -132,10 +106,25 @@ "runId": "kafka-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_topic,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_topic,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -144,7 +133,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "key_topic", + "schemaName": "key_value_topic", "platform": "urn:li:dataPlatform:kafka", "version": 0, "created": { @@ -155,10 +144,11 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "hash": "44fd7a7b325d6fdd4275b1f02a79c1a8", + "hash": "cc452cf58242cdb9d09cf33d657497d8", "platformSchema": { "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "", + "documentSchema": "{\"type\":\"record\",\"name\":\"CreateUserRequest\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Value schema for kafka topic\",\"fields\":[{\"name\":\"email\",\"type\":\"string\"},{\"name\":\"firstName\",\"type\":\"string\"},{\"name\":\"lastName\",\"type\":\"string\"}]}", + "documentSchemaType": "AVRO", "keySchema": "{\"type\":\"record\",\"name\":\"UserKey\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Key schema for kafka topic\",\"fields\":[{\"name\":\"id\",\"type\":\"long\"},{\"name\":\"namespace\",\"type\":\"string\"}]}", "keySchemaType": "AVRO" } @@ -187,6 +177,42 @@ "nativeDataType": "namespace", "recursive": false, "isPartOfKey": true + }, + { + "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "email", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].firstName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "firstName", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].lastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "lastName", + "recursive": false, + "isPartOfKey": false } ] } @@ -210,7 +236,8 @@ "max.message.bytes": "1048588", "unclean.leader.election.enable": "false" }, - "name": "key_topic", + "name": "key_value_topic", + "description": "Value schema for kafka topic", "tags": [] } } @@ -224,7 +251,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_topic,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -239,10 +266,42 @@ "runId": "kafka-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic,PROD)", + "changeType": "UPSERT", + "aspectName": "domains", + "aspect": { + "json": { + "domains": [ + "urn:li:domain:sales" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,value_topic,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -251,7 +310,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "key_value_topic", + "schemaName": "value_topic", "platform": "urn:li:dataPlatform:kafka", "version": 0, "created": { @@ -262,37 +321,25 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "hash": "cc452cf58242cdb9d09cf33d657497d8", + "hash": "dc1cf32c2688cc3d2d27fe6e856f06d2", "platformSchema": { "com.linkedin.pegasus2avro.schema.KafkaSchema": { "documentSchema": "{\"type\":\"record\",\"name\":\"CreateUserRequest\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Value schema for kafka topic\",\"fields\":[{\"name\":\"email\",\"type\":\"string\"},{\"name\":\"firstName\",\"type\":\"string\"},{\"name\":\"lastName\",\"type\":\"string\"}]}", "documentSchemaType": "AVRO", - "keySchema": "{\"type\":\"record\",\"name\":\"UserKey\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Key schema for kafka topic\",\"fields\":[{\"name\":\"id\",\"type\":\"long\"},{\"name\":\"namespace\",\"type\":\"string\"}]}", + "keySchema": "\"string\"", "keySchemaType": "AVRO" } }, "fields": [ { - "fieldPath": "[version=2.0].[key=True].[type=UserKey].[type=long].id", - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "id", - "recursive": false, - "isPartOfKey": true - }, - { - "fieldPath": "[version=2.0].[key=True].[type=UserKey].[type=string].namespace", + "fieldPath": "[version=2.0].[key=True].[type=string]", "nullable": false, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "namespace", + "nativeDataType": "string", "recursive": false, "isPartOfKey": true }, @@ -354,7 +401,7 @@ "max.message.bytes": "1048588", "unclean.leader.election.enable": "false" }, - "name": "key_value_topic", + "name": "value_topic", "description": "Value schema for kafka topic", "tags": [] } @@ -369,7 +416,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,value_topic,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -386,14 +433,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,value_topic,PROD)", "changeType": "UPSERT", - "aspectName": "domains", + "aspectName": "browsePathsV2", "aspect": { "json": { - "domains": [ - "urn:li:domain:sales" - ] + "path": [] } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/looker/golden_looker_mces.json b/metadata-ingestion/tests/integration/looker/golden_looker_mces.json index f3fbc3c459043..6167c63e6c9b8 100644 --- a/metadata-ingestion/tests/integration/looker/golden_looker_mces.json +++ b/metadata-ingestion/tests/integration/looker/golden_looker_mces.json @@ -43,8 +43,9 @@ "changeType": "UPSERT", "aspectName": "embed", "aspect": { - "value": "{\"renderUrl\": \"https://looker.company.com/embed/dashboards/11\"}", - "contentType": "application/json" + "json": { + "renderUrl": "https://looker.company.com/embed/dashboards/11" + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -57,8 +58,54 @@ "changeType": "UPSERT", "aspectName": "inputFields", "aspect": { - "value": "{\"fields\": [{\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)\", \"schemaField\": {\"fieldPath\": \"calc\", \"nullable\": false, \"description\": \"\", \"label\": \"foobar\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false}}, {\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)\", \"schemaField\": {\"fieldPath\": \"dim1\", \"nullable\": false, \"description\": \"dimension one description\", \"label\": \"Dimensions One Label\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Dimension\"}]}, \"isPartOfKey\": false}}]}", - "contentType": "application/json" + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)", + "schemaField": { + "fieldPath": "calc", + "nullable": false, + "description": "", + "label": "foobar", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)", + "schemaField": { + "fieldPath": "dim1", + "nullable": false, + "description": "dimension one description", + "label": "Dimensions One Label", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + } + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -71,8 +118,30 @@ "changeType": "UPSERT", "aspectName": "inputFields", "aspect": { - "value": "{\"fields\": [{\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.10),calc)\", \"schemaField\": {\"fieldPath\": \"calc\", \"nullable\": false, \"description\": \"\", \"label\": \"foobar\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false}}]}", - "contentType": "application/json" + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.10),calc)", + "schemaField": { + "fieldPath": "calc", + "nullable": false, + "description": "", + "label": "foobar", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [] + }, + "isPartOfKey": false + } + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -85,8 +154,74 @@ "changeType": "UPSERT", "aspectName": "inputFields", "aspect": { - "value": "{\"fields\": [{\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)\", \"schemaField\": {\"fieldPath\": \"calc\", \"nullable\": false, \"description\": \"\", \"label\": \"foobar\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false}}, {\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)\", \"schemaField\": {\"fieldPath\": \"dim1\", \"nullable\": false, \"description\": \"dimension one description\", \"label\": \"Dimensions One Label\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Dimension\"}]}, \"isPartOfKey\": false}}, {\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.10),calc)\", \"schemaField\": {\"fieldPath\": \"calc\", \"nullable\": false, \"description\": \"\", \"label\": \"foobar\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false}}]}", - "contentType": "application/json" + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)", + "schemaField": { + "fieldPath": "calc", + "nullable": false, + "description": "", + "label": "foobar", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)", + "schemaField": { + "fieldPath": "dim1", + "nullable": false, + "description": "dimension one description", + "label": "Dimensions One Label", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.10),calc)", + "schemaField": { + "fieldPath": "calc", + "nullable": false, + "description": "", + "label": "foobar", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [] + }, + "isPartOfKey": false + } + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -195,8 +330,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Explore\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "Explore" + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -209,8 +347,31 @@ "changeType": "UPSERT", "aspectName": "embed", "aspect": { - "value": "{\"renderUrl\": \"https://looker.company.com/embed/explore/data/my_view\"}", - "contentType": "application/json" + "json": { + "renderUrl": "https://looker.company.com/embed/explore/data/my_view" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "lkml_samples" + }, + { + "id": "explores" + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -319,8 +480,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Explore\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "Explore" + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -333,8 +497,31 @@ "changeType": "UPSERT", "aspectName": "embed", "aspect": { - "value": "{\"renderUrl\": \"https://looker.company.com/embed/explore/bogus data/my_view\"}", - "contentType": "application/json" + "json": { + "renderUrl": "https://looker.company.com/embed/explore/bogus data/my_view" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "lkml_samples" + }, + { + "id": "explores" + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -446,8 +633,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -460,8 +648,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -474,8 +663,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -488,8 +678,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -502,8 +693,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, diff --git a/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json index f4d53360d270b..e66ec4bb89d8c 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json @@ -43,8 +43,9 @@ "changeType": "UPSERT", "aspectName": "embed", "aspect": { - "value": "{\"renderUrl\": \"https://looker.company.com/embed/dashboards/11\"}", - "contentType": "application/json" + "json": { + "renderUrl": "https://looker.company.com/embed/dashboards/11" + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -57,8 +58,54 @@ "changeType": "UPSERT", "aspectName": "inputFields", "aspect": { - "value": "{\"fields\": [{\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)\", \"schemaField\": {\"fieldPath\": \"calc\", \"nullable\": false, \"description\": \"\", \"label\": \"foobar\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false}}, {\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)\", \"schemaField\": {\"fieldPath\": \"dim1\", \"nullable\": false, \"description\": \"dimension one description\", \"label\": \"Dimensions One Label\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Dimension\"}]}, \"isPartOfKey\": false}}]}", - "contentType": "application/json" + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)", + "schemaField": { + "fieldPath": "calc", + "nullable": false, + "description": "", + "label": "foobar", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)", + "schemaField": { + "fieldPath": "dim1", + "nullable": false, + "description": "dimension one description", + "label": "Dimensions One Label", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + } + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -71,8 +118,54 @@ "changeType": "UPSERT", "aspectName": "inputFields", "aspect": { - "value": "{\"fields\": [{\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)\", \"schemaField\": {\"fieldPath\": \"calc\", \"nullable\": false, \"description\": \"\", \"label\": \"foobar\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false}}, {\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)\", \"schemaField\": {\"fieldPath\": \"dim1\", \"nullable\": false, \"description\": \"dimension one description\", \"label\": \"Dimensions One Label\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Dimension\"}]}, \"isPartOfKey\": false}}]}", - "contentType": "application/json" + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)", + "schemaField": { + "fieldPath": "calc", + "nullable": false, + "description": "", + "label": "foobar", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)", + "schemaField": { + "fieldPath": "dim1", + "nullable": false, + "description": "dimension one description", + "label": "Dimensions One Label", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + } + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -181,8 +274,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Explore\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "Explore" + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -195,8 +291,31 @@ "changeType": "UPSERT", "aspectName": "embed", "aspect": { - "value": "{\"renderUrl\": \"https://looker.company.com/embed/explore/data/my_view\"}", - "contentType": "application/json" + "json": { + "renderUrl": "https://looker.company.com/embed/explore/data/my_view" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "lkml_samples" + }, + { + "id": "explores" + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -308,8 +427,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -322,8 +442,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -336,8 +457,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -350,8 +472,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, diff --git a/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json b/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json index 250532cb9dcbe..11e0760decae3 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json @@ -300,6 +300,28 @@ "runId": "looker-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker_hub" + }, + { + "id": "explores" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { diff --git a/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json index cbd8cc58b4423..ddfd102cb15b0 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json @@ -373,6 +373,28 @@ "runId": "looker-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "lkml_samples" + }, + { + "id": "explores" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -501,6 +523,28 @@ "runId": "looker-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,sales_model.explore.sales_explore,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "lkml_samples" + }, + { + "id": "explores" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest.json index 788e0fcf1cf40..54624986216b8 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest.json @@ -43,8 +43,9 @@ "changeType": "UPSERT", "aspectName": "embed", "aspect": { - "value": "{\"renderUrl\": \"https://looker.company.com/embed/dashboards/1\"}", - "contentType": "application/json" + "json": { + "renderUrl": "https://looker.company.com/embed/dashboards/1" + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -57,8 +58,54 @@ "changeType": "UPSERT", "aspectName": "inputFields", "aspect": { - "value": "{\"fields\": [{\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)\", \"schemaField\": {\"fieldPath\": \"calc\", \"nullable\": false, \"description\": \"\", \"label\": \"foobar\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false}}, {\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)\", \"schemaField\": {\"fieldPath\": \"dim1\", \"nullable\": false, \"description\": \"dimension one description\", \"label\": \"Dimensions One Label\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Dimension\"}]}, \"isPartOfKey\": false}}]}", - "contentType": "application/json" + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)", + "schemaField": { + "fieldPath": "calc", + "nullable": false, + "description": "", + "label": "foobar", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)", + "schemaField": { + "fieldPath": "dim1", + "nullable": false, + "description": "dimension one description", + "label": "Dimensions One Label", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + } + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -71,8 +118,54 @@ "changeType": "UPSERT", "aspectName": "inputFields", "aspect": { - "value": "{\"fields\": [{\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)\", \"schemaField\": {\"fieldPath\": \"calc\", \"nullable\": false, \"description\": \"\", \"label\": \"foobar\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false}}, {\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)\", \"schemaField\": {\"fieldPath\": \"dim1\", \"nullable\": false, \"description\": \"dimension one description\", \"label\": \"Dimensions One Label\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Dimension\"}]}, \"isPartOfKey\": false}}]}", - "contentType": "application/json" + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)", + "schemaField": { + "fieldPath": "calc", + "nullable": false, + "description": "", + "label": "foobar", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)", + "schemaField": { + "fieldPath": "dim1", + "nullable": false, + "description": "dimension one description", + "label": "Dimensions One Label", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + } + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -181,8 +274,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Explore\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "Explore" + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -195,8 +291,31 @@ "changeType": "UPSERT", "aspectName": "embed", "aspect": { - "value": "{\"renderUrl\": \"https://looker.company.com/embed/explore/data/my_view\"}", - "contentType": "application/json" + "json": { + "renderUrl": "https://looker.company.com/embed/explore/data/my_view" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "lkml_samples" + }, + { + "id": "explores" + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -308,8 +427,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -322,8 +442,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -336,8 +457,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -350,8 +472,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json index 2536f2290150f..6cab0db8c33cf 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json @@ -43,8 +43,9 @@ "changeType": "UPSERT", "aspectName": "embed", "aspect": { - "value": "{\"renderUrl\": \"https://looker.company.com/embed/dashboards/1\"}", - "contentType": "application/json" + "json": { + "renderUrl": "https://looker.company.com/embed/dashboards/1" + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -57,8 +58,54 @@ "changeType": "UPSERT", "aspectName": "inputFields", "aspect": { - "value": "{\"fields\": [{\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)\", \"schemaField\": {\"fieldPath\": \"calc\", \"nullable\": false, \"description\": \"\", \"label\": \"foobar\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false}}, {\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)\", \"schemaField\": {\"fieldPath\": \"dim1\", \"nullable\": false, \"description\": \"dimension one description\", \"label\": \"Dimensions One Label\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Dimension\"}]}, \"isPartOfKey\": false}}]}", - "contentType": "application/json" + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)", + "schemaField": { + "fieldPath": "calc", + "nullable": false, + "description": "", + "label": "foobar", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)", + "schemaField": { + "fieldPath": "dim1", + "nullable": false, + "description": "dimension one description", + "label": "Dimensions One Label", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + } + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -71,8 +118,54 @@ "changeType": "UPSERT", "aspectName": "inputFields", "aspect": { - "value": "{\"fields\": [{\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)\", \"schemaField\": {\"fieldPath\": \"calc\", \"nullable\": false, \"description\": \"\", \"label\": \"foobar\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false}}, {\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)\", \"schemaField\": {\"fieldPath\": \"dim1\", \"nullable\": false, \"description\": \"dimension one description\", \"label\": \"Dimensions One Label\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Dimension\"}]}, \"isPartOfKey\": false}}]}", - "contentType": "application/json" + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)", + "schemaField": { + "fieldPath": "calc", + "nullable": false, + "description": "", + "label": "foobar", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)", + "schemaField": { + "fieldPath": "dim1", + "nullable": false, + "description": "dimension one description", + "label": "Dimensions One Label", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + } + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -205,8 +298,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Explore\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "Explore" + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -219,8 +315,31 @@ "changeType": "UPSERT", "aspectName": "embed", "aspect": { - "value": "{\"renderUrl\": \"https://looker.company.com/embed/explore/data/my_view\"}", - "contentType": "application/json" + "json": { + "renderUrl": "https://looker.company.com/embed/explore/data/my_view" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "lkml_samples" + }, + { + "id": "explores" + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -332,8 +451,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -346,8 +466,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -360,8 +481,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -374,8 +496,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json index 80674911f09be..9a088a7a8baef 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json @@ -43,8 +43,9 @@ "changeType": "UPSERT", "aspectName": "embed", "aspect": { - "value": "{\"renderUrl\": \"https://looker.company.com/embed/dashboards/1\"}", - "contentType": "application/json" + "json": { + "renderUrl": "https://looker.company.com/embed/dashboards/1" + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -57,8 +58,54 @@ "changeType": "UPSERT", "aspectName": "inputFields", "aspect": { - "value": "{\"fields\": [{\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)\", \"schemaField\": {\"fieldPath\": \"calc\", \"nullable\": false, \"description\": \"\", \"label\": \"foobar\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false}}, {\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)\", \"schemaField\": {\"fieldPath\": \"dim1\", \"nullable\": false, \"description\": \"dimension one description\", \"label\": \"Dimensions One Label\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Dimension\"}]}, \"isPartOfKey\": false}}]}", - "contentType": "application/json" + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)", + "schemaField": { + "fieldPath": "calc", + "nullable": false, + "description": "", + "label": "foobar", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)", + "schemaField": { + "fieldPath": "dim1", + "nullable": false, + "description": "dimension one description", + "label": "Dimensions One Label", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + } + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -71,8 +118,54 @@ "changeType": "UPSERT", "aspectName": "inputFields", "aspect": { - "value": "{\"fields\": [{\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)\", \"schemaField\": {\"fieldPath\": \"calc\", \"nullable\": false, \"description\": \"\", \"label\": \"foobar\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false}}, {\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)\", \"schemaField\": {\"fieldPath\": \"dim1\", \"nullable\": false, \"description\": \"dimension one description\", \"label\": \"Dimensions One Label\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Dimension\"}]}, \"isPartOfKey\": false}}]}", - "contentType": "application/json" + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)", + "schemaField": { + "fieldPath": "calc", + "nullable": false, + "description": "", + "label": "foobar", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)", + "schemaField": { + "fieldPath": "dim1", + "nullable": false, + "description": "dimension one description", + "label": "Dimensions One Label", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + } + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -197,8 +290,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Explore\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "Explore" + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -211,8 +307,31 @@ "changeType": "UPSERT", "aspectName": "embed", "aspect": { - "value": "{\"renderUrl\": \"https://looker.company.com/embed/explore/data/my_view\"}", - "contentType": "application/json" + "json": { + "renderUrl": "https://looker.company.com/embed/explore/data/my_view" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "lkml_samples" + }, + { + "id": "explores" + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -324,8 +443,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -338,8 +458,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -352,8 +473,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -366,8 +488,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, diff --git a/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json b/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json index 374d4b69bbd6f..f8e2565e492e1 100644 --- a/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json +++ b/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json @@ -43,8 +43,9 @@ "changeType": "UPSERT", "aspectName": "embed", "aspect": { - "value": "{\"renderUrl\": \"https://looker.company.com/embed/dashboards/1\"}", - "contentType": "application/json" + "json": { + "renderUrl": "https://looker.company.com/embed/dashboards/1" + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -57,8 +58,54 @@ "changeType": "UPSERT", "aspectName": "inputFields", "aspect": { - "value": "{\"fields\": [{\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)\", \"schemaField\": {\"fieldPath\": \"calc\", \"nullable\": false, \"description\": \"\", \"label\": \"foobar\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false}}, {\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)\", \"schemaField\": {\"fieldPath\": \"dim1\", \"nullable\": false, \"description\": \"dimension one description\", \"label\": \"Dimensions One Label\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Dimension\"}]}, \"isPartOfKey\": false}}]}", - "contentType": "application/json" + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)", + "schemaField": { + "fieldPath": "calc", + "nullable": false, + "description": "", + "label": "foobar", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)", + "schemaField": { + "fieldPath": "dim1", + "nullable": false, + "description": "dimension one description", + "label": "Dimensions One Label", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + } + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -71,8 +118,54 @@ "changeType": "UPSERT", "aspectName": "inputFields", "aspect": { - "value": "{\"fields\": [{\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)\", \"schemaField\": {\"fieldPath\": \"calc\", \"nullable\": false, \"description\": \"\", \"label\": \"foobar\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false}}, {\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)\", \"schemaField\": {\"fieldPath\": \"dim1\", \"nullable\": false, \"description\": \"dimension one description\", \"label\": \"Dimensions One Label\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Dimension\"}]}, \"isPartOfKey\": false}}]}", - "contentType": "application/json" + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)", + "schemaField": { + "fieldPath": "calc", + "nullable": false, + "description": "", + "label": "foobar", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD),dim1)", + "schemaField": { + "fieldPath": "dim1", + "nullable": false, + "description": "dimension one description", + "label": "Dimensions One Label", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + } + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -181,8 +274,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Explore\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "Explore" + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -195,8 +291,31 @@ "changeType": "UPSERT", "aspectName": "embed", "aspect": { - "value": "{\"renderUrl\": \"https://looker.company.com/embed/explore/data/my_view\"}", - "contentType": "application/json" + "json": { + "renderUrl": "https://looker.company.com/embed/explore/data/my_view" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "lkml_samples" + }, + { + "id": "explores" + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -308,8 +427,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -322,8 +442,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -336,8 +457,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -350,8 +472,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -359,13 +482,14 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(looker,dashboards.11)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,dashboard_elements.10)", "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": true}", - "contentType": "application/json" + "json": { + "removed": true + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -378,8 +502,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": true}", - "contentType": "application/json" + "json": { + "removed": true + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -387,13 +512,14 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(looker,dashboard_elements.10)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(looker,dashboards.11)", "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": true}", - "contentType": "application/json" + "json": { + "removed": true + } }, "systemMetadata": { "lastObserved": 1586847600000, diff --git a/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json b/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json index 68f01840ca39f..32d4f7bc64ab4 100644 --- a/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json +++ b/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json @@ -43,8 +43,9 @@ "changeType": "UPSERT", "aspectName": "embed", "aspect": { - "value": "{\"renderUrl\": \"https://looker.company.com/embed/dashboards/1\"}", - "contentType": "application/json" + "json": { + "renderUrl": "https://looker.company.com/embed/dashboards/1" + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -57,8 +58,30 @@ "changeType": "UPSERT", "aspectName": "inputFields", "aspect": { - "value": "{\"fields\": [{\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)\", \"schemaField\": {\"fieldPath\": \"calc\", \"nullable\": false, \"description\": \"\", \"label\": \"foobar\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false}}]}", - "contentType": "application/json" + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)", + "schemaField": { + "fieldPath": "calc", + "nullable": false, + "description": "", + "label": "foobar", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [] + }, + "isPartOfKey": false + } + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -71,8 +94,30 @@ "changeType": "UPSERT", "aspectName": "inputFields", "aspect": { - "value": "{\"fields\": [{\"schemaFieldUrn\": \"urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)\", \"schemaField\": {\"fieldPath\": \"calc\", \"nullable\": false, \"description\": \"\", \"label\": \"foobar\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"string\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false}}]}", - "contentType": "application/json" + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(looker,dashboard_elements.2),calc)", + "schemaField": { + "fieldPath": "calc", + "nullable": false, + "description": "", + "label": "foobar", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [] + }, + "isPartOfKey": false + } + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -181,8 +226,11 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Explore\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "Explore" + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -195,8 +243,31 @@ "changeType": "UPSERT", "aspectName": "embed", "aspect": { - "value": "{\"renderUrl\": \"https://looker.company.com/embed/explore/data/my_view\"}", - "contentType": "application/json" + "json": { + "renderUrl": "https://looker.company.com/embed/explore/data/my_view" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "lkml_samples" + }, + { + "id": "explores" + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -308,8 +379,16 @@ "changeType": "UPSERT", "aspectName": "dashboardUsageStatistics", "aspect": { - "value": "{\"timestampMillis\": 1586847600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"viewsCount\": 25, \"favoritesCount\": 5, \"lastViewedAt\": 1586847600000}", - "contentType": "application/json" + "json": { + "timestampMillis": 1586847600000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "viewsCount": 25, + "favoritesCount": 5, + "lastViewedAt": 1586847600000 + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -322,8 +401,33 @@ "changeType": "UPSERT", "aspectName": "dashboardUsageStatistics", "aspect": { - "value": "{\"timestampMillis\": 1656979200000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"executionsCount\": 14, \"uniqueUserCount\": 1, \"userCounts\": [{\"user\": \"urn:li:corpuser:test-1@looker.com\", \"executionsCount\": 16, \"usageCount\": 16, \"userEmail\": \"test-1@looker.com\"}, {\"user\": \"urn:li:corpuser:test-2@looker.com\", \"executionsCount\": 14, \"usageCount\": 14, \"userEmail\": \"test-2@looker.com\"}]}", - "contentType": "application/json" + "json": { + "timestampMillis": 1656979200000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "executionsCount": 14, + "uniqueUserCount": 1, + "userCounts": [ + { + "user": "urn:li:corpuser:test-1@looker.com", + "executionsCount": 16, + "usageCount": 16, + "userEmail": "test-1@looker.com" + }, + { + "user": "urn:li:corpuser:test-2@looker.com", + "executionsCount": 14, + "usageCount": 14, + "userEmail": "test-2@looker.com" + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -336,8 +440,19 @@ "changeType": "UPSERT", "aspectName": "dashboardUsageStatistics", "aspect": { - "value": "{\"timestampMillis\": 1657065600000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"executionsCount\": 14, \"uniqueUserCount\": 1}", - "contentType": "application/json" + "json": { + "timestampMillis": 1657065600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "executionsCount": 14, + "uniqueUserCount": 1 + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -350,8 +465,27 @@ "changeType": "UPSERT", "aspectName": "dashboardUsageStatistics", "aspect": { - "value": "{\"timestampMillis\": 1657152000000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"executionsCount\": 5, \"uniqueUserCount\": 1, \"userCounts\": [{\"user\": \"urn:li:corpuser:test-1@looker.com\", \"executionsCount\": 5, \"usageCount\": 5, \"userEmail\": \"test-1@looker.com\"}]}", - "contentType": "application/json" + "json": { + "timestampMillis": 1657152000000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "executionsCount": 5, + "uniqueUserCount": 1, + "userCounts": [ + { + "user": "urn:li:corpuser:test-1@looker.com", + "executionsCount": 5, + "usageCount": 5, + "userEmail": "test-1@looker.com" + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -364,8 +498,14 @@ "changeType": "UPSERT", "aspectName": "chartUsageStatistics", "aspect": { - "value": "{\"timestampMillis\": 1586847600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"viewsCount\": 30}", - "contentType": "application/json" + "json": { + "timestampMillis": 1586847600000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "viewsCount": 30 + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -378,8 +518,28 @@ "changeType": "UPSERT", "aspectName": "chartUsageStatistics", "aspect": { - "value": "{\"timestampMillis\": 1656979200000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"viewsCount\": 10, \"userCounts\": [{\"user\": \"urn:li:corpuser:test-1@looker.com\", \"viewsCount\": 10}, {\"user\": \"urn:li:corpuser:test-2@looker.com\", \"viewsCount\": 20}]}", - "contentType": "application/json" + "json": { + "timestampMillis": 1656979200000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "viewsCount": 10, + "userCounts": [ + { + "user": "urn:li:corpuser:test-1@looker.com", + "viewsCount": 10 + }, + { + "user": "urn:li:corpuser:test-2@looker.com", + "viewsCount": 20 + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -392,8 +552,18 @@ "changeType": "UPSERT", "aspectName": "chartUsageStatistics", "aspect": { - "value": "{\"timestampMillis\": 1657065600000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"viewsCount\": 20}", - "contentType": "application/json" + "json": { + "timestampMillis": 1657065600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "viewsCount": 20 + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -406,8 +576,18 @@ "changeType": "UPSERT", "aspectName": "chartUsageStatistics", "aspect": { - "value": "{\"timestampMillis\": 1657152000000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"viewsCount\": 35}", - "contentType": "application/json" + "json": { + "timestampMillis": 1657152000000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "viewsCount": 35 + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -420,8 +600,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -434,8 +615,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -448,8 +630,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -462,8 +645,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, @@ -476,8 +660,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, diff --git a/metadata-ingestion/tests/integration/lookml/expected_output.json b/metadata-ingestion/tests/integration/lookml/expected_output.json index bc5cb11fd15ff..cdf520cc23a30 100644 --- a/metadata-ingestion/tests/integration/lookml/expected_output.json +++ b/metadata-ingestion/tests/integration/lookml/expected_output.json @@ -213,6 +213,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -406,6 +431,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -488,6 +538,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -570,6 +645,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -697,6 +797,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -779,6 +904,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -861,6 +1011,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1018,6 +1193,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1100,6 +1300,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1261,6 +1486,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1435,6 +1685,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1649,6 +1924,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1791,6 +2091,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "entityType": "tag", "entityUrn": "urn:li:tag:Dimension", diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json index bf70b9f196e01..73edecbe62205 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json @@ -213,6 +213,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -406,6 +431,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -488,6 +538,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -570,6 +645,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -697,6 +797,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -779,6 +904,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -861,6 +1011,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1018,6 +1193,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1100,6 +1300,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1261,6 +1486,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1435,6 +1685,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1649,6 +1924,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1791,6 +2091,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json index 76c1653ada245..9aa6a952c40b4 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json @@ -213,6 +213,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -406,6 +431,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -488,6 +538,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -570,6 +645,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -697,6 +797,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -779,6 +904,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -861,6 +1011,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1018,6 +1193,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1100,6 +1300,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1261,6 +1486,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1435,6 +1685,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1649,6 +1924,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1791,6 +2091,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json index e63b9f1c91cd9..6ce6d809ae8f5 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json @@ -199,6 +199,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -378,6 +403,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -460,6 +510,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -542,6 +617,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -669,6 +769,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -751,6 +876,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -833,6 +983,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -901,6 +1076,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -983,6 +1183,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1144,6 +1369,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1318,6 +1568,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1532,6 +1807,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1674,6 +1974,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_golden_deleted_stateful.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_golden_deleted_stateful.json index 434310b13ef05..1016d4e211458 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_golden_deleted_stateful.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_golden_deleted_stateful.json @@ -213,6 +213,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -387,6 +412,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "entityType": "tag", "entityUrn": "urn:li:tag:Dimension", @@ -434,7 +484,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -449,7 +499,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -464,7 +514,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -479,7 +529,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -494,7 +544,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -509,7 +559,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -524,7 +574,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -539,7 +589,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -554,7 +604,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -569,7 +619,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -584,7 +634,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json index 2fcb8ea3b4dd9..fc91c97a53003 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json @@ -213,6 +213,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -406,6 +431,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -488,6 +538,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -570,6 +645,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -697,6 +797,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -779,6 +904,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -861,6 +1011,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1018,6 +1193,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1100,6 +1300,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1261,6 +1486,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1435,6 +1685,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1649,6 +1924,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1791,6 +2091,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_deny_pattern.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_deny_pattern.json index dda7a653d60fd..8635a570c0621 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_deny_pattern.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_deny_pattern.json @@ -81,6 +81,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -163,6 +188,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -290,6 +340,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -372,6 +447,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -454,6 +554,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json index 5125dd5a31604..19168aa323142 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json @@ -213,6 +213,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -406,6 +431,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -488,6 +538,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -570,6 +645,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -697,6 +797,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -779,6 +904,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -861,6 +1011,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1018,6 +1193,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1100,6 +1300,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1261,6 +1486,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1435,6 +1685,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1649,6 +1924,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1791,6 +2091,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json index a179a52e3ada1..d4ced76a7475d 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json @@ -214,6 +214,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -408,6 +433,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -491,6 +541,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -574,6 +649,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -702,6 +802,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -785,6 +910,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -868,6 +1018,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1026,6 +1201,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1109,6 +1309,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1271,6 +1496,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1446,6 +1696,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1661,6 +1936,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1804,6 +2104,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json b/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json index 068762aca6ff2..2bae6452145df 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json @@ -213,6 +213,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -387,6 +412,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -601,6 +651,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { diff --git a/metadata-ingestion/tests/integration/lookml/refinement_include_order_golden.json b/metadata-ingestion/tests/integration/lookml/refinement_include_order_golden.json index 875b07647b166..a5c316f365d4b 100644 --- a/metadata-ingestion/tests/integration/lookml/refinement_include_order_golden.json +++ b/metadata-ingestion/tests/integration/lookml/refinement_include_order_golden.json @@ -256,6 +256,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_refinement_sample1.model_1.view.book,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_refinement_sample1" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -338,6 +363,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_refinement_sample1.model_1.view.extend_book,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_refinement_sample1" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -510,6 +560,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_refinement_sample1.model_1.view.order,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_refinement_sample1" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -703,6 +778,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_refinement_sample1.model_2.view.issue_history,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_refinement_sample1" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "entityType": "tag", "entityUrn": "urn:li:tag:Dimension", diff --git a/metadata-ingestion/tests/integration/lookml/refinements_ingestion_golden.json b/metadata-ingestion/tests/integration/lookml/refinements_ingestion_golden.json index 7907a7d1122ff..de303d50e7acd 100644 --- a/metadata-ingestion/tests/integration/lookml/refinements_ingestion_golden.json +++ b/metadata-ingestion/tests/integration/lookml/refinements_ingestion_golden.json @@ -213,6 +213,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -406,6 +431,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_derived_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -488,6 +538,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.include_able_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -570,6 +645,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.looker_events,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -697,6 +797,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.extending_looker_events,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -779,6 +904,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.autodetect_sql_name_based_on_view_name,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -861,6 +1011,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.test_include_external_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1018,6 +1193,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.fragment_derived_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1100,6 +1300,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.customer_facts,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1261,6 +1486,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.ability,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1467,6 +1717,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.owners,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1681,6 +1956,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.view_derived_explore,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1823,6 +2123,31 @@ "runId": "lookml-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.flights,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "looker" + }, + { + "id": "lkml_samples" + }, + { + "id": "views" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test" + } +}, { "entityType": "tag", "entityUrn": "urn:li:tag:Dimension", diff --git a/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json b/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json index 5b3af74da20b9..4aaefb48d33e1 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json +++ b/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json @@ -66,6 +66,21 @@ "runId": "mysql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0f72a1bc79da282eb614cc089c0ba302", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,dataCharmer.employees,PROD)", @@ -218,6 +233,26 @@ "runId": "mysql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,dataCharmer.employees,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0f72a1bc79da282eb614cc089c0ba302", + "urn": "urn:li:container:0f72a1bc79da282eb614cc089c0ba302" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,dataCharmer.salaries,PROD)", @@ -346,6 +381,26 @@ "runId": "mysql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,dataCharmer.salaries,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0f72a1bc79da282eb614cc089c0ba302", + "urn": "urn:li:container:0f72a1bc79da282eb614cc089c0ba302" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,dataCharmer.employees,PROD)", @@ -1001,6 +1056,21 @@ "runId": "mysql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:17751259af32dd0385cad799df608c40", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD)", @@ -1182,6 +1252,26 @@ "runId": "mysql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:17751259af32dd0385cad799df608c40", + "urn": "urn:li:container:17751259af32dd0385cad799df608c40" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD)", @@ -1365,6 +1455,26 @@ "runId": "mysql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:17751259af32dd0385cad799df608c40", + "urn": "urn:li:container:17751259af32dd0385cad799df608c40" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD)", @@ -1530,6 +1640,26 @@ "runId": "mysql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:17751259af32dd0385cad799df608c40", + "urn": "urn:li:container:17751259af32dd0385cad799df608c40" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", @@ -1597,6 +1727,21 @@ "runId": "mysql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", @@ -1749,6 +1894,26 @@ "runId": "mysql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", + "urn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", @@ -1877,6 +2042,26 @@ "runId": "mysql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", + "urn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", @@ -2118,6 +2303,21 @@ "runId": "mysql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:28176129fe1c0e526e1803250ec124ef", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,test_cases.myset,PROD)", @@ -2210,6 +2410,26 @@ "runId": "mysql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,test_cases.myset,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:28176129fe1c0e526e1803250ec124ef", + "urn": "urn:li:container:28176129fe1c0e526e1803250ec124ef" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,test_cases.test_empty,PROD)", @@ -2302,6 +2522,26 @@ "runId": "mysql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,test_cases.test_empty,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:28176129fe1c0e526e1803250ec124ef", + "urn": "urn:li:container:28176129fe1c0e526e1803250ec124ef" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,test_cases.myset,PROD)", diff --git a/metadata-ingestion/tests/integration/mysql/mysql_mces_with_db_golden.json b/metadata-ingestion/tests/integration/mysql/mysql_mces_with_db_golden.json index 499b09c7423cd..5cfba57247bd3 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_mces_with_db_golden.json +++ b/metadata-ingestion/tests/integration/mysql/mysql_mces_with_db_golden.json @@ -83,6 +83,21 @@ "runId": "mysql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", @@ -252,6 +267,26 @@ "runId": "mysql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", + "urn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", @@ -397,6 +432,26 @@ "runId": "mysql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", + "urn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", diff --git a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json index 98c6a9de223e1..1e59ce730e3c1 100644 --- a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json +++ b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json @@ -66,6 +66,21 @@ "runId": "oracle-2022_02_03-07_00_00" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:c093e810646c7ebc493237bb24a3538f", @@ -149,6 +164,26 @@ "runId": "oracle-2022_02_03-07_00_00" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c093e810646c7ebc493237bb24a3538f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", + "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.test1,PROD)", @@ -231,6 +266,30 @@ "runId": "oracle-2022_02_03-07_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.test1,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", + "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" + }, + { + "id": "urn:li:container:c093e810646c7ebc493237bb24a3538f", + "urn": "urn:li:container:c093e810646c7ebc493237bb24a3538f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.test2,PROD)", @@ -313,6 +372,30 @@ "runId": "oracle-2022_02_03-07_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.test2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", + "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" + }, + { + "id": "urn:li:container:c093e810646c7ebc493237bb24a3538f", + "urn": "urn:li:container:c093e810646c7ebc493237bb24a3538f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", @@ -396,6 +479,26 @@ "runId": "oracle-2022_02_03-07_00_00" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", + "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.test3,PROD)", @@ -478,6 +581,30 @@ "runId": "oracle-2022_02_03-07_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.test3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", + "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" + }, + { + "id": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", + "urn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.test4,PROD)", @@ -559,5 +686,29 @@ "lastObserved": 1643871600000, "runId": "oracle-2022_02_03-07_00_00" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.test4,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", + "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" + }, + { + "id": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", + "urn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json index ffe5cd9314871..6a21925f2f601 100644 --- a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json +++ b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json @@ -66,6 +66,21 @@ "runId": "oracle-2022_02_03-07_00_00" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:c093e810646c7ebc493237bb24a3538f", @@ -149,6 +164,26 @@ "runId": "oracle-2022_02_03-07_00_00" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c093e810646c7ebc493237bb24a3538f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", + "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.test1,PROD)", @@ -231,6 +266,30 @@ "runId": "oracle-2022_02_03-07_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.test1,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", + "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" + }, + { + "id": "urn:li:container:c093e810646c7ebc493237bb24a3538f", + "urn": "urn:li:container:c093e810646c7ebc493237bb24a3538f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.test2,PROD)", @@ -313,6 +372,30 @@ "runId": "oracle-2022_02_03-07_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.test2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", + "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" + }, + { + "id": "urn:li:container:c093e810646c7ebc493237bb24a3538f", + "urn": "urn:li:container:c093e810646c7ebc493237bb24a3538f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", @@ -396,6 +479,26 @@ "runId": "oracle-2022_02_03-07_00_00" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", + "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.test3,PROD)", @@ -478,6 +581,30 @@ "runId": "oracle-2022_02_03-07_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.test3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", + "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" + }, + { + "id": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", + "urn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.test4,PROD)", @@ -559,5 +686,29 @@ "lastObserved": 1643871600000, "runId": "oracle-2022_02_03-07_00_00" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.test4,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", + "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" + }, + { + "id": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", + "urn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json b/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json index 5d22ead40e1e0..535ce964c6058 100644 --- a/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json +++ b/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json @@ -66,6 +66,21 @@ "runId": "postgres-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0202f800c992262c01ae6bbd5ee313f7", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1646575200000, + "runId": "postgres-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:a208486b83be39fa411922e07701d984", @@ -149,6 +164,26 @@ "runId": "postgres-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a208486b83be39fa411922e07701d984", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0202f800c992262c01ae6bbd5ee313f7", + "urn": "urn:li:container:0202f800c992262c01ae6bbd5ee313f7" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1646575200000, + "runId": "postgres-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:a6097853edba03be190d99ece4b307ff", @@ -216,6 +251,21 @@ "runId": "postgres-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a6097853edba03be190d99ece4b307ff", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1646575200000, + "runId": "postgres-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525", @@ -299,6 +349,26 @@ "runId": "postgres-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:a6097853edba03be190d99ece4b307ff", + "urn": "urn:li:container:a6097853edba03be190d99ece4b307ff" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1646575200000, + "runId": "postgres-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)", @@ -567,6 +637,30 @@ "runId": "postgres-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:a6097853edba03be190d99ece4b307ff", + "urn": "urn:li:container:a6097853edba03be190d99ece4b307ff" + }, + { + "id": "urn:li:container:51904fc8cd5cc729bc630decff284525", + "urn": "urn:li:container:51904fc8cd5cc729bc630decff284525" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1646575200000, + "runId": "postgres-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)", diff --git a/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json b/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json index 19052a53aaff1..b8b7cd9bf3900 100644 --- a/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json +++ b/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json @@ -66,6 +66,21 @@ "runId": "postgres-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a6097853edba03be190d99ece4b307ff", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1646575200000, + "runId": "postgres-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525", @@ -149,6 +164,26 @@ "runId": "postgres-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:a6097853edba03be190d99ece4b307ff", + "urn": "urn:li:container:a6097853edba03be190d99ece4b307ff" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1646575200000, + "runId": "postgres-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)", @@ -416,5 +451,29 @@ "lastObserved": 1646575200000, "runId": "postgres-test" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:a6097853edba03be190d99ece4b307ff", + "urn": "urn:li:container:a6097853edba03be190d99ece4b307ff" + }, + { + "id": "urn:li:container:51904fc8cd5cc729bc630decff284525", + "urn": "urn:li:container:51904fc8cd5cc729bc630decff284525" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1646575200000, + "runId": "postgres-test" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json index 580641441c548..3a8f168ce3b4c 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json @@ -108,6 +108,25 @@ "runId": "powerbi-admin-api-disabled-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-admin-api-disabled-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", @@ -187,6 +206,25 @@ "runId": "powerbi-admin-api-disabled-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-admin-api-disabled-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", @@ -300,6 +338,25 @@ "runId": "powerbi-admin-api-disabled-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-admin-api-disabled-test" + } +}, { "entityType": "corpuser", "entityUrn": "urn:li:corpuser:users.User1@foo.com", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json index 14dcbcaea93c0..ceb2720c0962b 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json @@ -967,6 +967,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", @@ -1057,6 +1076,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", @@ -1170,6 +1208,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", @@ -1877,6 +1934,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "corpuser", "entityUrn": "urn:li:corpuser:users.User1@foo.com", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json index 33283143a468f..850816bf80807 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json @@ -65,6 +65,21 @@ "runId": "powerbi-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", @@ -151,6 +166,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", + "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", @@ -237,6 +272,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", + "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", @@ -323,6 +378,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", + "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", @@ -409,6 +484,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", + "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", @@ -495,6 +590,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", + "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", @@ -581,6 +696,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", + "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", @@ -667,6 +802,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", + "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", @@ -753,6 +908,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", + "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", @@ -839,6 +1014,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", + "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "corpuser", "entityUrn": "urn:li:corpuser:users.User1@foo.com", @@ -986,6 +1181,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", + "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", @@ -1088,6 +1303,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", + "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", @@ -1216,6 +1451,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", + "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", @@ -1947,6 +2202,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", + "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)", @@ -2046,6 +2321,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", + "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", @@ -2187,6 +2482,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", + "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "corpuser", "entityUrn": "urn:li:corpuser:users.User1@foo.com", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json index 05e8c0caf8b3c..5f896c2596f91 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json @@ -740,6 +740,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", @@ -827,6 +846,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", @@ -911,5 +949,24 @@ "lastObserved": 1643871600000, "runId": "powerbi-test" } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json index 6b99ee55e02eb..bac9d0d333fef 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json @@ -903,6 +903,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", @@ -990,6 +1009,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", @@ -1103,6 +1141,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "corpuser", "entityUrn": "urn:li:corpuser:users.User1@foo.com", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json index 3ac9c531a15a8..dcfaef2de84f8 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json @@ -770,6 +770,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", @@ -857,6 +876,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", @@ -970,6 +1008,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "corpuser", "entityUrn": "urn:li:corpuser:users.User1@foo.com", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json index 2145092429f59..5b746cbc8590d 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json @@ -946,6 +946,25 @@ "runId": "powerbi-lineage-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-lineage-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", @@ -1033,6 +1052,25 @@ "runId": "powerbi-lineage-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-lineage-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", @@ -1146,6 +1184,25 @@ "runId": "powerbi-lineage-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-lineage-test" + } +}, { "entityType": "corpuser", "entityUrn": "urn:li:corpuser:users.User1@foo.com", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json index b54ccaf79befd..fccce9dde1063 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json @@ -770,6 +770,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", @@ -857,6 +876,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", @@ -970,6 +1008,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "corpuser", "entityUrn": "urn:li:corpuser:users.User1@foo.com", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json index 1f0084380dfc8..a4527b9715704 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json @@ -65,6 +65,21 @@ "runId": "powerbi-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", @@ -226,6 +241,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc", + "urn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", @@ -387,6 +422,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc", + "urn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", @@ -600,6 +655,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc", + "urn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", @@ -695,6 +770,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3", + "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", @@ -800,6 +895,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3", + "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", @@ -900,6 +1015,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3", + "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", @@ -1034,6 +1169,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3", + "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc", @@ -1115,6 +1270,26 @@ "runId": "powerbi-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3", + "urn": "urn:li:container:e3dc21b5c79f9d594f639a9f57d7f2c3" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "tag", "entityUrn": "urn:li:tag:Certified", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json index 33e4635fedb3f..06decfe7d5b4a 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json @@ -770,6 +770,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", @@ -857,6 +876,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", @@ -970,6 +1008,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,aws-ap-south-1.dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "corpuser", "entityUrn": "urn:li:corpuser:users.User1@foo.com", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_report.json b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json index 7934f53f78f98..ee2d864214342 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_report.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json @@ -770,6 +770,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", @@ -857,6 +876,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", @@ -970,6 +1008,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", @@ -1581,6 +1638,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)", @@ -1665,6 +1741,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", @@ -1791,6 +1886,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "corpuser", "entityUrn": "urn:li:corpuser:users.User1@foo.com", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json index 0ce75fc5b8e61..0fb2457f2178b 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json @@ -740,6 +740,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", @@ -827,6 +846,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", @@ -912,6 +950,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)", @@ -993,5 +1050,24 @@ "lastObserved": 1643871600000, "runId": "powerbi-test" } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "second-demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json index 3553f79c8c8bb..4e258cffc4b4f 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json @@ -970,6 +970,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", @@ -1057,6 +1076,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", @@ -1170,6 +1208,25 @@ "runId": "powerbi-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, { "entityType": "corpuser", "entityUrn": "urn:li:corpuser:users.User1@foo.com", diff --git a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json index 298c45b4cee49..f3439e0e1770b 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json +++ b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json @@ -66,6 +66,21 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", @@ -149,6 +164,26 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", @@ -229,7 +264,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" }, @@ -262,6 +297,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", @@ -399,7 +458,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" }, @@ -432,6 +491,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", @@ -561,7 +644,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" }, @@ -594,6 +677,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", @@ -707,7 +814,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" }, @@ -741,6 +848,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", @@ -848,7 +979,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" }, @@ -881,6 +1012,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", @@ -958,7 +1113,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" }, @@ -991,6 +1146,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", @@ -1081,7 +1260,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", "partitioned_columns": "baz" @@ -1115,6 +1294,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", @@ -1234,6 +1437,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", @@ -1399,5 +1626,29 @@ "lastObserved": 1632398400000, "runId": "presto-on-hive-test" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json index ecef07a7d6ad9..61f4e1e9be1e9 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json +++ b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json @@ -66,6 +66,21 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", @@ -149,6 +164,26 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "urn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.map_test,PROD)", @@ -229,7 +264,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" }, @@ -262,6 +297,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "urn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab" + }, + { + "id": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "urn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.union_test,PROD)", @@ -399,7 +458,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" }, @@ -432,6 +491,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "urn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab" + }, + { + "id": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "urn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.nested_struct_test,PROD)", @@ -561,7 +644,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" }, @@ -594,6 +677,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "urn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab" + }, + { + "id": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "urn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test,PROD)", @@ -707,7 +814,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" }, @@ -741,6 +848,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "urn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab" + }, + { + "id": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "urn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.struct_test,PROD)", @@ -848,7 +979,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" }, @@ -881,6 +1012,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "urn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab" + }, + { + "id": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "urn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1._test_table_underscore,PROD)", @@ -958,7 +1113,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" }, @@ -991,6 +1146,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "urn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab" + }, + { + "id": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "urn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.pokes,PROD)", @@ -1081,7 +1260,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", "partitioned_columns": "baz" @@ -1115,6 +1294,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "urn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab" + }, + { + "id": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "urn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test_presto_view,PROD)", @@ -1246,5 +1449,29 @@ "lastObserved": 1632398400000, "runId": "presto-on-hive-test" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "urn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab" + }, + { + "id": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "urn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_3.json b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_3.json index a2ac5cb5ee788..b1b3bf93d6ed3 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_3.json +++ b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_3.json @@ -66,6 +66,21 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", @@ -149,6 +164,26 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.map_test,PROD)", @@ -229,7 +264,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" }, @@ -262,6 +297,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.union_test,PROD)", @@ -399,7 +458,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" }, @@ -432,6 +491,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.nested_struct_test,PROD)", @@ -561,7 +644,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" }, @@ -594,6 +677,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test,PROD)", @@ -707,7 +814,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" }, @@ -741,6 +848,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.struct_test,PROD)", @@ -848,7 +979,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" }, @@ -881,6 +1012,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1._test_table_underscore,PROD)", @@ -958,7 +1113,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" }, @@ -991,6 +1146,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.pokes,PROD)", @@ -1081,7 +1260,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", "partitioned_columns": "baz" @@ -1115,6 +1294,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_presto_view,PROD)", @@ -1234,6 +1437,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_view,PROD)", @@ -1399,5 +1626,29 @@ "lastObserved": 1632398400000, "runId": "presto-on-hive-test" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_4.json b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_4.json index cabd42dbf25ab..5a41a32cbf0e2 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_4.json +++ b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_4.json @@ -66,6 +66,21 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", @@ -149,6 +164,26 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "urn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.map_test,PROD)", @@ -229,7 +264,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" }, @@ -262,6 +297,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "urn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab" + }, + { + "id": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "urn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.union_test,PROD)", @@ -399,7 +458,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" }, @@ -432,6 +491,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "urn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab" + }, + { + "id": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "urn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.nested_struct_test,PROD)", @@ -561,7 +644,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" }, @@ -594,6 +677,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "urn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab" + }, + { + "id": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "urn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test,PROD)", @@ -707,7 +814,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" }, @@ -741,6 +848,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "urn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab" + }, + { + "id": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "urn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.struct_test,PROD)", @@ -848,7 +979,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" }, @@ -881,6 +1012,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "urn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab" + }, + { + "id": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "urn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1._test_table_underscore,PROD)", @@ -958,7 +1113,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" }, @@ -991,6 +1146,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "urn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab" + }, + { + "id": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "urn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.pokes,PROD)", @@ -1081,7 +1260,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-05-12", + "create_date": "2023-07-05", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", "partitioned_columns": "baz" @@ -1115,6 +1294,30 @@ "runId": "presto-on-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "urn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab" + }, + { + "id": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "urn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test_presto_view,PROD)", @@ -1246,5 +1449,29 @@ "lastObserved": 1632398400000, "runId": "presto-on-hive-test" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "urn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab" + }, + { + "id": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "urn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_file_without_extension.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_file_without_extension.json index 30cd27016b145..818a8b7a5adde 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_file_without_extension.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_file_without_extension.json @@ -127,7 +127,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684577564017 + "lastUpdatedTimestamp": 1688445129021 } }, "systemMetadata": { @@ -204,6 +204,21 @@ "runId": "file_without_extension.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "file_without_extension.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", @@ -288,6 +303,26 @@ "runId": "file_without_extension.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "file_without_extension.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", @@ -372,6 +407,30 @@ "runId": "file_without_extension.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + }, + { + "id": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "urn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "file_without_extension.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:c46207c164682005e865a54fcf7f4a9f", @@ -456,6 +515,34 @@ "runId": "file_without_extension.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c46207c164682005e865a54fcf7f4a9f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + }, + { + "id": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "urn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999" + }, + { + "id": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", + "urn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "file_without_extension.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:fd50ce59cb982671fc700636ab5744e2", @@ -540,6 +627,38 @@ "runId": "file_without_extension.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fd50ce59cb982671fc700636ab5744e2", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + }, + { + "id": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "urn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999" + }, + { + "id": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", + "urn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db" + }, + { + "id": "urn:li:container:c46207c164682005e865a54fcf7f4a9f", + "urn": "urn:li:container:c46207c164682005e865a54fcf7f4a9f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "file_without_extension.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1", @@ -624,6 +743,42 @@ "runId": "file_without_extension.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + }, + { + "id": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "urn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999" + }, + { + "id": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", + "urn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db" + }, + { + "id": "urn:li:container:c46207c164682005e865a54fcf7f4a9f", + "urn": "urn:li:container:c46207c164682005e865a54fcf7f4a9f" + }, + { + "id": "urn:li:container:fd50ce59cb982671fc700636ab5744e2", + "urn": "urn:li:container:fd50ce59cb982671fc700636ab5744e2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "file_without_extension.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb", @@ -708,6 +863,46 @@ "runId": "file_without_extension.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + }, + { + "id": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "urn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999" + }, + { + "id": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", + "urn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db" + }, + { + "id": "urn:li:container:c46207c164682005e865a54fcf7f4a9f", + "urn": "urn:li:container:c46207c164682005e865a54fcf7f4a9f" + }, + { + "id": "urn:li:container:fd50ce59cb982671fc700636ab5744e2", + "urn": "urn:li:container:fd50ce59cb982671fc700636ab5744e2" + }, + { + "id": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1", + "urn": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "file_without_extension.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:46c2438600873ee3264c24c4ac6081b9", @@ -792,6 +987,50 @@ "runId": "file_without_extension.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46c2438600873ee3264c24c4ac6081b9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + }, + { + "id": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "urn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999" + }, + { + "id": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", + "urn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db" + }, + { + "id": "urn:li:container:c46207c164682005e865a54fcf7f4a9f", + "urn": "urn:li:container:c46207c164682005e865a54fcf7f4a9f" + }, + { + "id": "urn:li:container:fd50ce59cb982671fc700636ab5744e2", + "urn": "urn:li:container:fd50ce59cb982671fc700636ab5744e2" + }, + { + "id": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1", + "urn": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1" + }, + { + "id": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb", + "urn": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "file_without_extension.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:0715c65c4403307618fe7203ff768eaa", @@ -876,6 +1115,54 @@ "runId": "file_without_extension.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0715c65c4403307618fe7203ff768eaa", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + }, + { + "id": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "urn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999" + }, + { + "id": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", + "urn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db" + }, + { + "id": "urn:li:container:c46207c164682005e865a54fcf7f4a9f", + "urn": "urn:li:container:c46207c164682005e865a54fcf7f4a9f" + }, + { + "id": "urn:li:container:fd50ce59cb982671fc700636ab5744e2", + "urn": "urn:li:container:fd50ce59cb982671fc700636ab5744e2" + }, + { + "id": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1", + "urn": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1" + }, + { + "id": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb", + "urn": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb" + }, + { + "id": "urn:li:container:46c2438600873ee3264c24c4ac6081b9", + "urn": "urn:li:container:46c2438600873ee3264c24c4ac6081b9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "file_without_extension.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/no_extension/small,DEV)", @@ -905,5 +1192,57 @@ "lastObserved": 1615443388097, "runId": "file_without_extension.json" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/no_extension/small,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + }, + { + "id": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "urn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999" + }, + { + "id": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", + "urn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db" + }, + { + "id": "urn:li:container:c46207c164682005e865a54fcf7f4a9f", + "urn": "urn:li:container:c46207c164682005e865a54fcf7f4a9f" + }, + { + "id": "urn:li:container:fd50ce59cb982671fc700636ab5744e2", + "urn": "urn:li:container:fd50ce59cb982671fc700636ab5744e2" + }, + { + "id": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1", + "urn": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1" + }, + { + "id": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb", + "urn": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb" + }, + { + "id": "urn:li:container:46c2438600873ee3264c24c4ac6081b9", + "urn": "urn:li:container:46c2438600873ee3264c24c4ac6081b9" + }, + { + "id": "urn:li:container:0715c65c4403307618fe7203ff768eaa", + "urn": "urn:li:container:0715c65c4403307618fe7203ff768eaa" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "file_without_extension.json" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition.json index 0fac506fffeb6..51a8ef60849a9 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition.json @@ -127,7 +127,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684577524017 + "lastUpdatedTimestamp": 1688445089021 } }, "systemMetadata": { @@ -202,6 +202,21 @@ "runId": "folder_no_partition.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", @@ -284,6 +299,26 @@ "runId": "folder_no_partition.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", @@ -366,6 +401,30 @@ "runId": "folder_no_partition.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", @@ -448,6 +507,34 @@ "runId": "folder_no_partition.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", @@ -530,6 +617,38 @@ "runId": "folder_no_partition.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", @@ -612,6 +731,42 @@ "runId": "folder_no_partition.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", @@ -694,6 +849,46 @@ "runId": "folder_no_partition.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", @@ -776,6 +971,50 @@ "runId": "folder_no_partition.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", @@ -1095,6 +1334,54 @@ "runId": "folder_no_partition.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + }, + { + "id": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "urn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", @@ -1223,7 +1510,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684577554017 + "lastUpdatedTimestamp": 1688445119021 } }, "systemMetadata": { @@ -1550,6 +1837,54 @@ "runId": "folder_no_partition.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + }, + { + "id": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "urn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_exclude.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_exclude.json index 76332ea656436..153ff7799987b 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_exclude.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_exclude.json @@ -127,7 +127,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684577554017 + "lastUpdatedTimestamp": 1688445119021 } }, "systemMetadata": { @@ -202,6 +202,21 @@ "runId": "folder_no_partition_exclude.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", @@ -284,6 +299,26 @@ "runId": "folder_no_partition_exclude.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", @@ -366,6 +401,30 @@ "runId": "folder_no_partition_exclude.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", @@ -448,6 +507,34 @@ "runId": "folder_no_partition_exclude.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", @@ -530,6 +617,38 @@ "runId": "folder_no_partition_exclude.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", @@ -612,6 +731,42 @@ "runId": "folder_no_partition_exclude.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", @@ -694,6 +849,46 @@ "runId": "folder_no_partition_exclude.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", @@ -776,6 +971,50 @@ "runId": "folder_no_partition_exclude.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", @@ -1109,5 +1348,53 @@ "lastObserved": 1615443388097, "runId": "folder_no_partition_exclude.json" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + }, + { + "id": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "urn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_filename.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_filename.json index 8ce5efc13bfa3..8e67bbf10bd6d 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_filename.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_filename.json @@ -127,7 +127,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684577524017 + "lastUpdatedTimestamp": 1688445089021 } }, "systemMetadata": { @@ -202,6 +202,21 @@ "runId": "folder_no_partition_filename.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", @@ -284,6 +299,26 @@ "runId": "folder_no_partition_filename.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", @@ -366,6 +401,30 @@ "runId": "folder_no_partition_filename.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", @@ -448,6 +507,34 @@ "runId": "folder_no_partition_filename.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", @@ -530,6 +617,38 @@ "runId": "folder_no_partition_filename.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", @@ -612,6 +731,42 @@ "runId": "folder_no_partition_filename.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", @@ -694,6 +849,46 @@ "runId": "folder_no_partition_filename.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", @@ -776,6 +971,50 @@ "runId": "folder_no_partition_filename.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", @@ -1095,6 +1334,54 @@ "runId": "folder_no_partition_filename.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + }, + { + "id": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "urn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", @@ -1223,7 +1510,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684577554017 + "lastUpdatedTimestamp": 1688445119021 } }, "systemMetadata": { @@ -1550,6 +1837,54 @@ "runId": "folder_no_partition_filename.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + }, + { + "id": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "urn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_glob.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_glob.json index 8b6e37a2c2935..6878e6f618233 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_glob.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_glob.json @@ -127,7 +127,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684577554017 + "lastUpdatedTimestamp": 1688445119021 } }, "systemMetadata": { @@ -202,6 +202,21 @@ "runId": "folder_no_partition_glob.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", @@ -284,6 +299,26 @@ "runId": "folder_no_partition_glob.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", @@ -366,6 +401,30 @@ "runId": "folder_no_partition_glob.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", @@ -448,6 +507,34 @@ "runId": "folder_no_partition_glob.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", @@ -530,6 +617,38 @@ "runId": "folder_no_partition_glob.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", @@ -612,6 +731,42 @@ "runId": "folder_no_partition_glob.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", @@ -694,6 +849,46 @@ "runId": "folder_no_partition_glob.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", @@ -776,6 +971,50 @@ "runId": "folder_no_partition_glob.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", @@ -1109,5 +1348,53 @@ "lastObserved": 1615443388097, "runId": "folder_no_partition_glob.json" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + }, + { + "id": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "urn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_basic.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_basic.json index 80b447da190f6..b7025839833e1 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_basic.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_basic.json @@ -559,7 +559,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1683874985593 + "lastUpdatedTimestamp": 1688445259021 } }, "systemMetadata": { @@ -634,6 +634,21 @@ "runId": "folder_partition_basic.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", @@ -716,6 +731,26 @@ "runId": "folder_partition_basic.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", @@ -798,6 +833,30 @@ "runId": "folder_partition_basic.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", @@ -880,6 +939,34 @@ "runId": "folder_partition_basic.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", @@ -962,6 +1049,38 @@ "runId": "folder_partition_basic.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", @@ -1044,6 +1163,42 @@ "runId": "folder_partition_basic.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", @@ -1126,6 +1281,46 @@ "runId": "folder_partition_basic.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", @@ -1208,6 +1403,50 @@ "runId": "folder_partition_basic.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", @@ -1362,5 +1601,53 @@ "lastObserved": 1615443388097, "runId": "folder_partition_basic.json" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + }, + { + "id": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "urn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_keyval.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_keyval.json index 992149c4e3276..e94bf407b9fda 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_keyval.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_keyval.json @@ -559,7 +559,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1683874985593 + "lastUpdatedTimestamp": 1688445259021 } }, "systemMetadata": { @@ -634,6 +634,21 @@ "runId": "folder_partition_keyval.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", @@ -716,6 +731,26 @@ "runId": "folder_partition_keyval.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", @@ -798,6 +833,30 @@ "runId": "folder_partition_keyval.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", @@ -880,6 +939,34 @@ "runId": "folder_partition_keyval.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", @@ -962,6 +1049,38 @@ "runId": "folder_partition_keyval.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", @@ -1044,6 +1163,42 @@ "runId": "folder_partition_keyval.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", @@ -1126,6 +1281,46 @@ "runId": "folder_partition_keyval.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", @@ -1208,6 +1403,50 @@ "runId": "folder_partition_keyval.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", @@ -1362,5 +1601,53 @@ "lastObserved": 1615443388097, "runId": "folder_partition_keyval.json" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + }, + { + "id": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "urn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_update_schema.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_update_schema.json index 3ab908f7f2be3..3aa5813b74c23 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_update_schema.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_update_schema.json @@ -559,7 +559,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1683874985593 + "lastUpdatedTimestamp": 1688445259021 } }, "systemMetadata": { @@ -634,6 +634,21 @@ "runId": "folder_partition_update_schema.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", @@ -716,6 +731,26 @@ "runId": "folder_partition_update_schema.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", @@ -798,6 +833,30 @@ "runId": "folder_partition_update_schema.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", @@ -880,6 +939,34 @@ "runId": "folder_partition_update_schema.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", @@ -962,6 +1049,38 @@ "runId": "folder_partition_update_schema.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", @@ -1044,6 +1163,42 @@ "runId": "folder_partition_update_schema.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", @@ -1126,6 +1281,46 @@ "runId": "folder_partition_update_schema.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", @@ -1208,6 +1403,50 @@ "runId": "folder_partition_update_schema.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", @@ -1362,5 +1601,53 @@ "lastObserved": 1615443388097, "runId": "folder_partition_update_schema.json" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + }, + { + "id": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "urn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema.json" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_files.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_files.json index b70aefd58f518..7fc7e79a643b7 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_files.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_files.json @@ -439,7 +439,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684577324017 + "lastUpdatedTimestamp": 1688444889021 } }, "systemMetadata": { @@ -516,6 +516,21 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", @@ -600,6 +615,26 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", @@ -684,6 +719,30 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + }, + { + "id": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "urn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:c46207c164682005e865a54fcf7f4a9f", @@ -768,6 +827,34 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c46207c164682005e865a54fcf7f4a9f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + }, + { + "id": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "urn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999" + }, + { + "id": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", + "urn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:fd50ce59cb982671fc700636ab5744e2", @@ -852,6 +939,38 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fd50ce59cb982671fc700636ab5744e2", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + }, + { + "id": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "urn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999" + }, + { + "id": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", + "urn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db" + }, + { + "id": "urn:li:container:c46207c164682005e865a54fcf7f4a9f", + "urn": "urn:li:container:c46207c164682005e865a54fcf7f4a9f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1", @@ -936,6 +1055,42 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + }, + { + "id": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "urn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999" + }, + { + "id": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", + "urn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db" + }, + { + "id": "urn:li:container:c46207c164682005e865a54fcf7f4a9f", + "urn": "urn:li:container:c46207c164682005e865a54fcf7f4a9f" + }, + { + "id": "urn:li:container:fd50ce59cb982671fc700636ab5744e2", + "urn": "urn:li:container:fd50ce59cb982671fc700636ab5744e2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb", @@ -1020,6 +1175,46 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + }, + { + "id": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "urn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999" + }, + { + "id": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", + "urn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db" + }, + { + "id": "urn:li:container:c46207c164682005e865a54fcf7f4a9f", + "urn": "urn:li:container:c46207c164682005e865a54fcf7f4a9f" + }, + { + "id": "urn:li:container:fd50ce59cb982671fc700636ab5744e2", + "urn": "urn:li:container:fd50ce59cb982671fc700636ab5744e2" + }, + { + "id": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1", + "urn": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:46c2438600873ee3264c24c4ac6081b9", @@ -1104,6 +1299,50 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46c2438600873ee3264c24c4ac6081b9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + }, + { + "id": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "urn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999" + }, + { + "id": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", + "urn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db" + }, + { + "id": "urn:li:container:c46207c164682005e865a54fcf7f4a9f", + "urn": "urn:li:container:c46207c164682005e865a54fcf7f4a9f" + }, + { + "id": "urn:li:container:fd50ce59cb982671fc700636ab5744e2", + "urn": "urn:li:container:fd50ce59cb982671fc700636ab5744e2" + }, + { + "id": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1", + "urn": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1" + }, + { + "id": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb", + "urn": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv,DEV)", @@ -2416,6 +2655,54 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + }, + { + "id": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "urn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999" + }, + { + "id": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", + "urn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db" + }, + { + "id": "urn:li:container:c46207c164682005e865a54fcf7f4a9f", + "urn": "urn:li:container:c46207c164682005e865a54fcf7f4a9f" + }, + { + "id": "urn:li:container:fd50ce59cb982671fc700636ab5744e2", + "urn": "urn:li:container:fd50ce59cb982671fc700636ab5744e2" + }, + { + "id": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1", + "urn": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1" + }, + { + "id": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb", + "urn": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb" + }, + { + "id": "urn:li:container:46c2438600873ee3264c24c4ac6081b9", + "urn": "urn:li:container:46c2438600873ee3264c24c4ac6081b9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,DEV)", @@ -2544,7 +2831,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684577334017 + "lastUpdatedTimestamp": 1688444899021 } }, "systemMetadata": { @@ -2901,6 +3188,54 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + }, + { + "id": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "urn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999" + }, + { + "id": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", + "urn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db" + }, + { + "id": "urn:li:container:c46207c164682005e865a54fcf7f4a9f", + "urn": "urn:li:container:c46207c164682005e865a54fcf7f4a9f" + }, + { + "id": "urn:li:container:fd50ce59cb982671fc700636ab5744e2", + "urn": "urn:li:container:fd50ce59cb982671fc700636ab5744e2" + }, + { + "id": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1", + "urn": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1" + }, + { + "id": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb", + "urn": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb" + }, + { + "id": "urn:li:container:46c2438600873ee3264c24c4ac6081b9", + "urn": "urn:li:container:46c2438600873ee3264c24c4ac6081b9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", @@ -3029,7 +3364,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684577344017 + "lastUpdatedTimestamp": 1688444909021 } }, "systemMetadata": { @@ -3402,6 +3737,54 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + }, + { + "id": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "urn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999" + }, + { + "id": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", + "urn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db" + }, + { + "id": "urn:li:container:c46207c164682005e865a54fcf7f4a9f", + "urn": "urn:li:container:c46207c164682005e865a54fcf7f4a9f" + }, + { + "id": "urn:li:container:fd50ce59cb982671fc700636ab5744e2", + "urn": "urn:li:container:fd50ce59cb982671fc700636ab5744e2" + }, + { + "id": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1", + "urn": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1" + }, + { + "id": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb", + "urn": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb" + }, + { + "id": "urn:li:container:46c2438600873ee3264c24c4ac6081b9", + "urn": "urn:li:container:46c2438600873ee3264c24c4ac6081b9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/countries_json.json,DEV)", @@ -3506,7 +3889,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684577354017 + "lastUpdatedTimestamp": 1688444919021 } }, "systemMetadata": { @@ -3562,6 +3945,54 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/countries_json.json,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + }, + { + "id": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "urn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999" + }, + { + "id": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", + "urn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db" + }, + { + "id": "urn:li:container:c46207c164682005e865a54fcf7f4a9f", + "urn": "urn:li:container:c46207c164682005e865a54fcf7f4a9f" + }, + { + "id": "urn:li:container:fd50ce59cb982671fc700636ab5744e2", + "urn": "urn:li:container:fd50ce59cb982671fc700636ab5744e2" + }, + { + "id": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1", + "urn": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1" + }, + { + "id": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb", + "urn": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb" + }, + { + "id": "urn:li:container:46c2438600873ee3264c24c4ac6081b9", + "urn": "urn:li:container:46c2438600873ee3264c24c4ac6081b9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet.parquet,DEV)", @@ -3690,7 +4121,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684577364017 + "lastUpdatedTimestamp": 1688444929021 } }, "systemMetadata": { @@ -4017,6 +4448,54 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet.parquet,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + }, + { + "id": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "urn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999" + }, + { + "id": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", + "urn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db" + }, + { + "id": "urn:li:container:c46207c164682005e865a54fcf7f4a9f", + "urn": "urn:li:container:c46207c164682005e865a54fcf7f4a9f" + }, + { + "id": "urn:li:container:fd50ce59cb982671fc700636ab5744e2", + "urn": "urn:li:container:fd50ce59cb982671fc700636ab5744e2" + }, + { + "id": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1", + "urn": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1" + }, + { + "id": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb", + "urn": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb" + }, + { + "id": "urn:li:container:46c2438600873ee3264c24c4ac6081b9", + "urn": "urn:li:container:46c2438600873ee3264c24c4ac6081b9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/small.csv,DEV)", @@ -4145,7 +4624,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684577374017 + "lastUpdatedTimestamp": 1688444939021 } }, "systemMetadata": { @@ -4331,6 +4810,54 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/small.csv,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + }, + { + "id": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "urn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999" + }, + { + "id": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", + "urn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db" + }, + { + "id": "urn:li:container:c46207c164682005e865a54fcf7f4a9f", + "urn": "urn:li:container:c46207c164682005e865a54fcf7f4a9f" + }, + { + "id": "urn:li:container:fd50ce59cb982671fc700636ab5744e2", + "urn": "urn:li:container:fd50ce59cb982671fc700636ab5744e2" + }, + { + "id": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1", + "urn": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1" + }, + { + "id": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb", + "urn": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb" + }, + { + "id": "urn:li:container:46c2438600873ee3264c24c4ac6081b9", + "urn": "urn:li:container:46c2438600873ee3264c24c4ac6081b9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", @@ -4771,7 +5298,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684577384017 + "lastUpdatedTimestamp": 1688444949021 } }, "systemMetadata": { @@ -6954,6 +7481,54 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", + "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" + }, + { + "id": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999", + "urn": "urn:li:container:09bc75f9aaf92d57502aad33cab2e999" + }, + { + "id": "urn:li:container:564adc1710f345e4777dbdc81a4b20db", + "urn": "urn:li:container:564adc1710f345e4777dbdc81a4b20db" + }, + { + "id": "urn:li:container:c46207c164682005e865a54fcf7f4a9f", + "urn": "urn:li:container:c46207c164682005e865a54fcf7f4a9f" + }, + { + "id": "urn:li:container:fd50ce59cb982671fc700636ab5744e2", + "urn": "urn:li:container:fd50ce59cb982671fc700636ab5744e2" + }, + { + "id": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1", + "urn": "urn:li:container:34dcc9e05fe0d390619cbe1210771ba1" + }, + { + "id": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb", + "urn": "urn:li:container:ec0a322960f194cdd055a5a6d5172ecb" + }, + { + "id": "urn:li:container:46c2438600873ee3264c24c4ac6081b9", + "urn": "urn:li:container:46c2438600873ee3264c24c4ac6081b9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv,DEV)", diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_spec_for_files.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_spec_for_files.json index 24c524fe3f30b..1bd75ae457cb4 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_spec_for_files.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_spec_for_files.json @@ -127,7 +127,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684577334017 + "lastUpdatedTimestamp": 1688444899021 } }, "systemMetadata": { @@ -202,6 +202,21 @@ "runId": "multiple_spec_for_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_spec_for_files.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", @@ -284,6 +299,26 @@ "runId": "multiple_spec_for_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_spec_for_files.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", @@ -366,6 +401,30 @@ "runId": "multiple_spec_for_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_spec_for_files.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:0cfba651f28915437b9ca20b860794b8", @@ -448,6 +507,34 @@ "runId": "multiple_spec_for_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + }, + { + "id": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "urn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_spec_for_files.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", @@ -530,6 +617,38 @@ "runId": "multiple_spec_for_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + }, + { + "id": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "urn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc" + }, + { + "id": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "urn": "urn:li:container:0cfba651f28915437b9ca20b860794b8" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_spec_for_files.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd", @@ -612,6 +731,42 @@ "runId": "multiple_spec_for_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + }, + { + "id": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "urn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc" + }, + { + "id": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "urn": "urn:li:container:0cfba651f28915437b9ca20b860794b8" + }, + { + "id": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", + "urn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_spec_for_files.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b", @@ -694,6 +849,46 @@ "runId": "multiple_spec_for_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + }, + { + "id": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "urn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc" + }, + { + "id": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "urn": "urn:li:container:0cfba651f28915437b9ca20b860794b8" + }, + { + "id": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", + "urn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b" + }, + { + "id": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd", + "urn": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_spec_for_files.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:0b11f7e4b6df7e17a294482039b93289", @@ -776,6 +971,50 @@ "runId": "multiple_spec_for_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0b11f7e4b6df7e17a294482039b93289", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + }, + { + "id": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "urn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc" + }, + { + "id": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "urn": "urn:li:container:0cfba651f28915437b9ca20b860794b8" + }, + { + "id": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", + "urn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b" + }, + { + "id": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd", + "urn": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd" + }, + { + "id": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b", + "urn": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_spec_for_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", @@ -1125,6 +1364,54 @@ "runId": "multiple_spec_for_files.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + }, + { + "id": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "urn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc" + }, + { + "id": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "urn": "urn:li:container:0cfba651f28915437b9ca20b860794b8" + }, + { + "id": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", + "urn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b" + }, + { + "id": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd", + "urn": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd" + }, + { + "id": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b", + "urn": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b" + }, + { + "id": "urn:li:container:0b11f7e4b6df7e17a294482039b93289", + "urn": "urn:li:container:0b11f7e4b6df7e17a294482039b93289" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_spec_for_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,PROD)", @@ -1253,7 +1540,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684577344017 + "lastUpdatedTimestamp": 1688444909021 } }, "systemMetadata": { @@ -1626,6 +1913,54 @@ "runId": "multiple_spec_for_files.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + }, + { + "id": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "urn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc" + }, + { + "id": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "urn": "urn:li:container:0cfba651f28915437b9ca20b860794b8" + }, + { + "id": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", + "urn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b" + }, + { + "id": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd", + "urn": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd" + }, + { + "id": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b", + "urn": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b" + }, + { + "id": "urn:li:container:0b11f7e4b6df7e17a294482039b93289", + "urn": "urn:li:container:0b11f7e4b6df7e17a294482039b93289" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_spec_for_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_specs_of_different_buckets.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_specs_of_different_buckets.json index ffbe0a7422469..b9687b97571cb 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_specs_of_different_buckets.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_specs_of_different_buckets.json @@ -127,7 +127,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684577334017 + "lastUpdatedTimestamp": 1688444899021 } }, "systemMetadata": { @@ -202,6 +202,21 @@ "runId": "multiple_specs_of_different_buckets.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", @@ -284,6 +299,26 @@ "runId": "multiple_specs_of_different_buckets.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", @@ -366,6 +401,30 @@ "runId": "multiple_specs_of_different_buckets.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:0cfba651f28915437b9ca20b860794b8", @@ -448,6 +507,34 @@ "runId": "multiple_specs_of_different_buckets.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + }, + { + "id": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "urn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", @@ -530,6 +617,38 @@ "runId": "multiple_specs_of_different_buckets.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + }, + { + "id": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "urn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc" + }, + { + "id": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "urn": "urn:li:container:0cfba651f28915437b9ca20b860794b8" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd", @@ -612,6 +731,42 @@ "runId": "multiple_specs_of_different_buckets.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + }, + { + "id": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "urn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc" + }, + { + "id": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "urn": "urn:li:container:0cfba651f28915437b9ca20b860794b8" + }, + { + "id": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", + "urn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b", @@ -694,6 +849,46 @@ "runId": "multiple_specs_of_different_buckets.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + }, + { + "id": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "urn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc" + }, + { + "id": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "urn": "urn:li:container:0cfba651f28915437b9ca20b860794b8" + }, + { + "id": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", + "urn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b" + }, + { + "id": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd", + "urn": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:0b11f7e4b6df7e17a294482039b93289", @@ -776,6 +971,50 @@ "runId": "multiple_specs_of_different_buckets.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0b11f7e4b6df7e17a294482039b93289", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + }, + { + "id": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "urn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc" + }, + { + "id": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "urn": "urn:li:container:0cfba651f28915437b9ca20b860794b8" + }, + { + "id": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", + "urn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b" + }, + { + "id": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd", + "urn": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd" + }, + { + "id": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b", + "urn": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", @@ -1125,6 +1364,54 @@ "runId": "multiple_specs_of_different_buckets.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + }, + { + "id": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "urn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc" + }, + { + "id": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "urn": "urn:li:container:0cfba651f28915437b9ca20b860794b8" + }, + { + "id": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", + "urn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b" + }, + { + "id": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd", + "urn": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd" + }, + { + "id": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b", + "urn": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b" + }, + { + "id": "urn:li:container:0b11f7e4b6df7e17a294482039b93289", + "urn": "urn:li:container:0b11f7e4b6df7e17a294482039b93289" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,PROD)", @@ -1253,7 +1540,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684577344017 + "lastUpdatedTimestamp": 1688444909021 } }, "systemMetadata": { @@ -1626,6 +1913,54 @@ "runId": "multiple_specs_of_different_buckets.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + }, + { + "id": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "urn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc" + }, + { + "id": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "urn": "urn:li:container:0cfba651f28915437b9ca20b860794b8" + }, + { + "id": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", + "urn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b" + }, + { + "id": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd", + "urn": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd" + }, + { + "id": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b", + "urn": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b" + }, + { + "id": "urn:li:container:0b11f7e4b6df7e17a294482039b93289", + "urn": "urn:li:container:0b11f7e4b6df7e17a294482039b93289" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_single_file.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_single_file.json index dfe49f2fb0e9a..a5a68777cad5c 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_single_file.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_single_file.json @@ -127,7 +127,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684577334017 + "lastUpdatedTimestamp": 1688444899021 } }, "systemMetadata": { @@ -202,6 +202,21 @@ "runId": "single_file.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", @@ -284,6 +299,26 @@ "runId": "single_file.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", @@ -366,6 +401,30 @@ "runId": "single_file.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:0cfba651f28915437b9ca20b860794b8", @@ -448,6 +507,34 @@ "runId": "single_file.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + }, + { + "id": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "urn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", @@ -530,6 +617,38 @@ "runId": "single_file.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + }, + { + "id": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "urn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc" + }, + { + "id": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "urn": "urn:li:container:0cfba651f28915437b9ca20b860794b8" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd", @@ -612,6 +731,42 @@ "runId": "single_file.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + }, + { + "id": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "urn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc" + }, + { + "id": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "urn": "urn:li:container:0cfba651f28915437b9ca20b860794b8" + }, + { + "id": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", + "urn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b", @@ -694,6 +849,46 @@ "runId": "single_file.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + }, + { + "id": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "urn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc" + }, + { + "id": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "urn": "urn:li:container:0cfba651f28915437b9ca20b860794b8" + }, + { + "id": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", + "urn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b" + }, + { + "id": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd", + "urn": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:0b11f7e4b6df7e17a294482039b93289", @@ -776,6 +971,50 @@ "runId": "single_file.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0b11f7e4b6df7e17a294482039b93289", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + }, + { + "id": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "urn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc" + }, + { + "id": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "urn": "urn:li:container:0cfba651f28915437b9ca20b860794b8" + }, + { + "id": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", + "urn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b" + }, + { + "id": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd", + "urn": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd" + }, + { + "id": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b", + "urn": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", @@ -1139,5 +1378,53 @@ "lastObserved": 1615443388097, "runId": "single_file.json" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "urn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43" + }, + { + "id": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "urn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55" + }, + { + "id": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "urn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc" + }, + { + "id": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "urn": "urn:li:container:0cfba651f28915437b9ca20b860794b8" + }, + { + "id": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", + "urn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b" + }, + { + "id": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd", + "urn": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd" + }, + { + "id": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b", + "urn": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b" + }, + { + "id": "urn:li:container:0b11f7e4b6df7e17a294482039b93289", + "urn": "urn:li:container:0b11f7e4b6df7e17a294482039b93289" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json index 8f9d2646eb07c..5518e93600843 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json @@ -127,7 +127,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684552113000 + "lastUpdatedTimestamp": 1688419674000 } }, "systemMetadata": { @@ -204,6 +204,21 @@ "runId": "file_without_extension.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "file_without_extension.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", @@ -288,6 +303,26 @@ "runId": "file_without_extension.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "file_without_extension.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", @@ -372,6 +407,30 @@ "runId": "file_without_extension.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" + }, + { + "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "file_without_extension.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", @@ -456,6 +515,34 @@ "runId": "file_without_extension.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" + }, + { + "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" + }, + { + "id": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "urn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "file_without_extension.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:9b4624d58669059c9e62afb3d7341944", @@ -540,6 +627,38 @@ "runId": "file_without_extension.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9b4624d58669059c9e62afb3d7341944", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" + }, + { + "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" + }, + { + "id": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "urn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc" + }, + { + "id": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "urn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "file_without_extension.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/no_extension/small,DEV)", @@ -569,5 +688,41 @@ "lastObserved": 1615443388097, "runId": "file_without_extension.json" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/no_extension/small,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" + }, + { + "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" + }, + { + "id": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "urn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc" + }, + { + "id": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "urn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" + }, + { + "id": "urn:li:container:9b4624d58669059c9e62afb3d7341944", + "urn": "urn:li:container:9b4624d58669059c9e62afb3d7341944" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "file_without_extension.json" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json index f690c12500c5f..1b2ef998bd5ba 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json @@ -125,7 +125,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684552113000 + "lastUpdatedTimestamp": 1688419673000 } }, "systemMetadata": { @@ -200,6 +200,21 @@ "runId": "folder_no_partition.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", @@ -282,6 +297,26 @@ "runId": "folder_no_partition.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", @@ -364,6 +399,30 @@ "runId": "folder_no_partition.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", @@ -446,6 +505,34 @@ "runId": "folder_no_partition.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", @@ -461,6 +548,38 @@ "runId": "folder_no_partition.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + }, + { + "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", @@ -587,7 +706,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684552113000 + "lastUpdatedTimestamp": 1688419671000 } }, "systemMetadata": { @@ -610,6 +729,38 @@ "runId": "folder_no_partition.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + }, + { + "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json index 4dd1b7ef2409f..fe50a19546653 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json @@ -125,7 +125,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684552113000 + "lastUpdatedTimestamp": 1688419671000 } }, "systemMetadata": { @@ -200,6 +200,21 @@ "runId": "folder_no_partition_exclude.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", @@ -282,6 +297,26 @@ "runId": "folder_no_partition_exclude.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", @@ -364,6 +399,30 @@ "runId": "folder_no_partition_exclude.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", @@ -446,6 +505,34 @@ "runId": "folder_no_partition_exclude.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", @@ -475,5 +562,37 @@ "lastObserved": 1615443388097, "runId": "folder_no_partition_exclude.json" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + }, + { + "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json index db2bd170a7171..6a1b339b2e188 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json @@ -125,7 +125,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684552113000 + "lastUpdatedTimestamp": 1688419673000 } }, "systemMetadata": { @@ -200,6 +200,21 @@ "runId": "folder_no_partition_filename.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", @@ -282,6 +297,26 @@ "runId": "folder_no_partition_filename.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", @@ -364,6 +399,30 @@ "runId": "folder_no_partition_filename.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", @@ -446,6 +505,34 @@ "runId": "folder_no_partition_filename.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", @@ -461,6 +548,38 @@ "runId": "folder_no_partition_filename.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + }, + { + "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", @@ -587,7 +706,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684552113000 + "lastUpdatedTimestamp": 1688419671000 } }, "systemMetadata": { @@ -610,6 +729,38 @@ "runId": "folder_no_partition_filename.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + }, + { + "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json index 78b055553743f..071ef3efffcea 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json @@ -125,7 +125,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684552113000 + "lastUpdatedTimestamp": 1688419671000 } }, "systemMetadata": { @@ -200,6 +200,21 @@ "runId": "folder_no_partition_glob.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", @@ -282,6 +297,26 @@ "runId": "folder_no_partition_glob.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", @@ -364,6 +399,30 @@ "runId": "folder_no_partition_glob.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", @@ -446,6 +505,34 @@ "runId": "folder_no_partition_glob.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", @@ -475,5 +562,37 @@ "lastObserved": 1615443388097, "runId": "folder_no_partition_glob.json" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + }, + { + "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json index c7744d54dc1eb..188369c92db46 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json @@ -557,7 +557,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1683887704000 + "lastUpdatedTimestamp": 1688419673000 } }, "systemMetadata": { @@ -632,6 +632,21 @@ "runId": "folder_partition_basic.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", @@ -714,6 +729,26 @@ "runId": "folder_partition_basic.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", @@ -796,6 +831,30 @@ "runId": "folder_partition_basic.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", @@ -878,6 +937,34 @@ "runId": "folder_partition_basic.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", @@ -907,5 +994,37 @@ "lastObserved": 1615443388097, "runId": "folder_partition_basic.json" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + }, + { + "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json index 73dc7d3d3ac8c..5ab5f043e9621 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json @@ -557,7 +557,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1683887704000 + "lastUpdatedTimestamp": 1688419673000 } }, "systemMetadata": { @@ -632,6 +632,21 @@ "runId": "folder_partition_keyval.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", @@ -714,6 +729,26 @@ "runId": "folder_partition_keyval.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", @@ -796,6 +831,30 @@ "runId": "folder_partition_keyval.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", @@ -878,6 +937,34 @@ "runId": "folder_partition_keyval.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", @@ -907,5 +994,37 @@ "lastObserved": 1615443388097, "runId": "folder_partition_keyval.json" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + }, + { + "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json index 4c60f002380af..6e42b68f1fa95 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json @@ -557,7 +557,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1683887704000 + "lastUpdatedTimestamp": 1688419673000 } }, "systemMetadata": { @@ -632,6 +632,21 @@ "runId": "folder_partition_update_schema.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", @@ -714,6 +729,26 @@ "runId": "folder_partition_update_schema.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", @@ -796,6 +831,30 @@ "runId": "folder_partition_update_schema.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", @@ -878,6 +937,34 @@ "runId": "folder_partition_update_schema.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", @@ -907,5 +994,37 @@ "lastObserved": 1615443388097, "runId": "folder_partition_update_schema.json" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + }, + { + "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema.json" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json index 7f8081c589e47..01ff0030e33c9 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json @@ -439,7 +439,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684552113000 + "lastUpdatedTimestamp": 1688419669000 } }, "systemMetadata": { @@ -516,6 +516,21 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", @@ -600,6 +615,26 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", @@ -684,6 +719,30 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" + }, + { + "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", @@ -768,6 +827,34 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" + }, + { + "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" + }, + { + "id": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "urn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv,DEV)", @@ -783,6 +870,38 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" + }, + { + "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" + }, + { + "id": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "urn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc" + }, + { + "id": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "urn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,DEV)", @@ -911,7 +1030,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684552113000 + "lastUpdatedTimestamp": 1688419668000 } }, "systemMetadata": { @@ -934,6 +1053,38 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" + }, + { + "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" + }, + { + "id": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "urn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc" + }, + { + "id": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "urn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", @@ -1062,7 +1213,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684552113000 + "lastUpdatedTimestamp": 1688419668000 } }, "systemMetadata": { @@ -1085,6 +1236,38 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" + }, + { + "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" + }, + { + "id": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "urn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc" + }, + { + "id": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "urn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/countries_json.json,DEV)", @@ -1189,7 +1372,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684552113000 + "lastUpdatedTimestamp": 1688419668000 } }, "systemMetadata": { @@ -1212,6 +1395,38 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/countries_json.json,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" + }, + { + "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" + }, + { + "id": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "urn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc" + }, + { + "id": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "urn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet.parquet,DEV)", @@ -1340,7 +1555,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684552113000 + "lastUpdatedTimestamp": 1688419668000 } }, "systemMetadata": { @@ -1363,6 +1578,38 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet.parquet,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" + }, + { + "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" + }, + { + "id": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "urn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc" + }, + { + "id": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "urn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/small.csv,DEV)", @@ -1491,7 +1738,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684552113000 + "lastUpdatedTimestamp": 1688419668000 } }, "systemMetadata": { @@ -1514,6 +1761,38 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/small.csv,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" + }, + { + "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" + }, + { + "id": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "urn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc" + }, + { + "id": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "urn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", @@ -1954,7 +2233,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684552113000 + "lastUpdatedTimestamp": 1688419668000 } }, "systemMetadata": { @@ -1977,6 +2256,38 @@ "runId": "multiple_files.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" + }, + { + "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" + }, + { + "id": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "urn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc" + }, + { + "id": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "urn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv,DEV)", diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_spec_for_files.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_spec_for_files.json index 5a7eac70a01ef..26d1636eae852 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_spec_for_files.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_spec_for_files.json @@ -149,7 +149,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684552113000 + "lastUpdatedTimestamp": 1688419668000 } }, "systemMetadata": { @@ -224,6 +224,21 @@ "runId": "multiple_spec_for_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_spec_for_files.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", @@ -306,6 +321,26 @@ "runId": "multiple_spec_for_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "urn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_spec_for_files.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", @@ -388,6 +423,30 @@ "runId": "multiple_spec_for_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "urn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" + }, + { + "id": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "urn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_spec_for_files.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", @@ -470,6 +529,34 @@ "runId": "multiple_spec_for_files.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "urn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" + }, + { + "id": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "urn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c" + }, + { + "id": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "urn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_spec_for_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", @@ -485,6 +572,38 @@ "runId": "multiple_spec_for_files.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "urn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" + }, + { + "id": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "urn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c" + }, + { + "id": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "urn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15" + }, + { + "id": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", + "urn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_spec_for_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,PROD)", @@ -635,7 +754,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684552113000 + "lastUpdatedTimestamp": 1688419668000 } }, "systemMetadata": { @@ -658,6 +777,38 @@ "runId": "multiple_spec_for_files.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "urn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" + }, + { + "id": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "urn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c" + }, + { + "id": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "urn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15" + }, + { + "id": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", + "urn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_spec_for_files.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json index c014d7564100f..d7a831e9490ba 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json @@ -149,7 +149,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684552113000 + "lastUpdatedTimestamp": 1688419668000 } }, "systemMetadata": { @@ -224,6 +224,21 @@ "runId": "multiple_specs_of_different_buckets.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", @@ -306,6 +321,26 @@ "runId": "multiple_specs_of_different_buckets.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "urn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", @@ -388,6 +423,30 @@ "runId": "multiple_specs_of_different_buckets.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "urn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" + }, + { + "id": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "urn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", @@ -470,6 +529,34 @@ "runId": "multiple_specs_of_different_buckets.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "urn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" + }, + { + "id": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "urn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c" + }, + { + "id": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "urn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", @@ -485,6 +572,38 @@ "runId": "multiple_specs_of_different_buckets.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "urn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" + }, + { + "id": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "urn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c" + }, + { + "id": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "urn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15" + }, + { + "id": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", + "urn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket-2/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,PROD)", @@ -635,7 +754,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684552113000 + "lastUpdatedTimestamp": 1688419674000 } }, "systemMetadata": { @@ -710,6 +829,21 @@ "runId": "multiple_specs_of_different_buckets.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:680e54d5e3a7705caa1d99893fab4924", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f6d8484efac8152d10620c6c0699d02d", @@ -792,6 +926,26 @@ "runId": "multiple_specs_of_different_buckets.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6d8484efac8152d10620c6c0699d02d", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:680e54d5e3a7705caa1d99893fab4924", + "urn": "urn:li:container:680e54d5e3a7705caa1d99893fab4924" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:6e8c28494477b4a90cf5fd395217bae0", @@ -874,6 +1028,30 @@ "runId": "multiple_specs_of_different_buckets.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:6e8c28494477b4a90cf5fd395217bae0", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:680e54d5e3a7705caa1d99893fab4924", + "urn": "urn:li:container:680e54d5e3a7705caa1d99893fab4924" + }, + { + "id": "urn:li:container:f6d8484efac8152d10620c6c0699d02d", + "urn": "urn:li:container:f6d8484efac8152d10620c6c0699d02d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:ce2eca2107ef4c0b47a8f4a65eff971c", @@ -956,6 +1134,34 @@ "runId": "multiple_specs_of_different_buckets.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ce2eca2107ef4c0b47a8f4a65eff971c", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:680e54d5e3a7705caa1d99893fab4924", + "urn": "urn:li:container:680e54d5e3a7705caa1d99893fab4924" + }, + { + "id": "urn:li:container:f6d8484efac8152d10620c6c0699d02d", + "urn": "urn:li:container:f6d8484efac8152d10620c6c0699d02d" + }, + { + "id": "urn:li:container:6e8c28494477b4a90cf5fd395217bae0", + "urn": "urn:li:container:6e8c28494477b4a90cf5fd395217bae0" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket-2/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,PROD)", @@ -986,6 +1192,38 @@ "runId": "multiple_specs_of_different_buckets.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket-2/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:680e54d5e3a7705caa1d99893fab4924", + "urn": "urn:li:container:680e54d5e3a7705caa1d99893fab4924" + }, + { + "id": "urn:li:container:f6d8484efac8152d10620c6c0699d02d", + "urn": "urn:li:container:f6d8484efac8152d10620c6c0699d02d" + }, + { + "id": "urn:li:container:6e8c28494477b4a90cf5fd395217bae0", + "urn": "urn:li:container:6e8c28494477b4a90cf5fd395217bae0" + }, + { + "id": "urn:li:container:ce2eca2107ef4c0b47a8f4a65eff971c", + "urn": "urn:li:container:ce2eca2107ef4c0b47a8f4a65eff971c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json index 9ac238b00a383..caf989fa376cf 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json @@ -149,7 +149,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1684552113000 + "lastUpdatedTimestamp": 1688419668000 } }, "systemMetadata": { @@ -224,6 +224,21 @@ "runId": "single_file.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", @@ -306,6 +321,26 @@ "runId": "single_file.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "urn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", @@ -388,6 +423,30 @@ "runId": "single_file.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "urn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" + }, + { + "id": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "urn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", @@ -470,6 +529,34 @@ "runId": "single_file.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "urn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" + }, + { + "id": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "urn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c" + }, + { + "id": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "urn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", @@ -500,6 +587,38 @@ "runId": "single_file.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "urn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" + }, + { + "id": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "urn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c" + }, + { + "id": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "urn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15" + }, + { + "id": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", + "urn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json" + } +}, { "entityType": "tag", "entityUrn": "urn:li:tag:baz:bob", diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json index 5fe9cb97e6cb7..59f096af2419c 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json @@ -93,6 +93,21 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "tag", "entityUrn": "urn:li:tag:other_db.other_schema.my_other_tag:other", @@ -219,6 +234,26 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", @@ -488,6 +523,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", @@ -757,6 +816,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", @@ -1026,6 +1109,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", @@ -1295,6 +1402,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", @@ -1564,6 +1695,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", @@ -1833,6 +1988,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", @@ -2102,6 +2281,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", @@ -2371,6 +2574,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", @@ -2640,6 +2867,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", @@ -2909,6 +3160,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "tag", "entityUrn": "urn:li:tag:test_db.test_schema.security:pii", @@ -3168,6 +3443,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "tag", "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_0:my_value_0", @@ -3477,6 +3776,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", @@ -5110,7 +5433,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", "changeType": "UPSERT", "aspectName": "datasetUsageStatistics", "aspect": { @@ -5138,7 +5461,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", "changeType": "UPSERT", "aspectName": "datasetUsageStatistics", "aspect": { @@ -5166,7 +5489,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "changeType": "UPSERT", "aspectName": "datasetUsageStatistics", "aspect": { @@ -5194,7 +5517,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", "changeType": "UPSERT", "aspectName": "datasetUsageStatistics", "aspect": { @@ -5222,7 +5545,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", "changeType": "UPSERT", "aspectName": "datasetUsageStatistics", "aspect": { @@ -5250,7 +5573,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", "changeType": "UPSERT", "aspectName": "datasetUsageStatistics", "aspect": { @@ -5278,7 +5601,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", "changeType": "UPSERT", "aspectName": "datasetUsageStatistics", "aspect": { @@ -5306,7 +5629,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", "changeType": "UPSERT", "aspectName": "datasetUsageStatistics", "aspect": { @@ -5334,7 +5657,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", "changeType": "UPSERT", "aspectName": "datasetUsageStatistics", "aspect": { @@ -5362,7 +5685,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", "changeType": "UPSERT", "aspectName": "datasetUsageStatistics", "aspect": { @@ -5390,7 +5713,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", "changeType": "UPSERT", "aspectName": "datasetUsageStatistics", "aspect": { @@ -5418,7 +5741,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", "changeType": "UPSERT", "aspectName": "datasetUsageStatistics", "aspect": { diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json index 6ec9af03b7bdb..21c2c656dff46 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json @@ -74,6 +74,21 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", @@ -165,6 +180,26 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", @@ -401,6 +436,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", @@ -637,6 +696,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", @@ -873,6 +956,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", @@ -1109,6 +1216,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", @@ -1345,6 +1476,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", @@ -1581,6 +1736,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", @@ -1817,6 +1996,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", @@ -2053,6 +2256,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", @@ -2289,6 +2516,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", @@ -2525,6 +2776,30 @@ "runId": "snowflake-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json index 03286eecfb5e4..be4ae9e047aea 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json @@ -66,6 +66,21 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", @@ -149,6 +164,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", @@ -232,6 +267,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", @@ -315,6 +370,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", @@ -398,6 +473,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", @@ -481,6 +576,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", @@ -564,6 +679,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", @@ -647,6 +782,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", @@ -730,6 +885,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", @@ -813,6 +988,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", @@ -896,6 +1091,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)", @@ -1000,6 +1215,30 @@ "runId": "mssql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", + "urn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", @@ -1083,6 +1322,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)", @@ -1188,6 +1447,30 @@ "runId": "mssql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)", @@ -1317,6 +1600,30 @@ "runId": "mssql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)", @@ -1434,20 +1741,44 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "browsePathsV2", "aspect": { "json": { - "customProperties": { - "platform": "mssql", - "env": "PROD", - "database": "demodata", - "schema": "guest" - }, - "name": "guest" - } + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "mssql", + "env": "PROD", + "database": "demodata", + "schema": "guest" + }, + "name": "guest" + } }, "systemMetadata": { "lastObserved": 1615443388097, @@ -1516,6 +1847,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", @@ -1599,6 +1950,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", @@ -1682,6 +2053,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", @@ -1749,6 +2140,21 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:47217386c89d8b94153f6ee31e7e77ec", @@ -1832,6 +2238,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:47217386c89d8b94153f6ee31e7e77ec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:5eb0d61efa998d1ccd5cbdc6ce4bb4af", @@ -1915,6 +2341,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5eb0d61efa998d1ccd5cbdc6ce4bb4af", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:2816b2cb7f90d3dce64125ba89fb1fa8", @@ -1998,6 +2444,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2816b2cb7f90d3dce64125ba89fb1fa8", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:20d0f0c94e9796ff44ff32d4d0e19084", @@ -2081,6 +2547,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:20d0f0c94e9796ff44ff32d4d0e19084", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:3600d2ebb33b25dac607624d7eae7575", @@ -2164,6 +2650,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3600d2ebb33b25dac607624d7eae7575", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:280f2e3aefacc346d0ce1590ec337c7d", @@ -2247,6 +2753,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:280f2e3aefacc346d0ce1590ec337c7d", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:cba5c3ca7f028fcf749593be369d3c24", @@ -2330,6 +2856,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:cba5c3ca7f028fcf749593be369d3c24", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:58c30fa72f213ca7e12fb04f5a7d150f", @@ -2413,6 +2959,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:58c30fa72f213ca7e12fb04f5a7d150f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:9387ddfeb7b57672cabd761ade89c49c", @@ -2496,6 +3062,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9387ddfeb7b57672cabd761ade89c49c", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:3a5f70e0e34834d4eeeb4d5a5caf03d0", @@ -2579,6 +3165,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3a5f70e0e34834d4eeeb4d5a5caf03d0", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.dbo.ProductsNew,PROD)", @@ -2683,6 +3289,30 @@ "runId": "mssql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.dbo.ProductsNew,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + }, + { + "id": "urn:li:container:3a5f70e0e34834d4eeeb4d5a5caf03d0", + "urn": "urn:li:container:3a5f70e0e34834d4eeeb4d5a5caf03d0" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:7cc43e5b4e2a7f2f66f1df774d1a0c63", @@ -2766,6 +3396,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7cc43e5b4e2a7f2f66f1df774d1a0c63", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.ItemsNew,PROD)", @@ -2870,6 +3520,30 @@ "runId": "mssql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.ItemsNew,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + }, + { + "id": "urn:li:container:7cc43e5b4e2a7f2f66f1df774d1a0c63", + "urn": "urn:li:container:7cc43e5b4e2a7f2f66f1df774d1a0c63" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.PersonsNew,PROD)", @@ -2998,6 +3672,30 @@ "runId": "mssql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.PersonsNew,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + }, + { + "id": "urn:li:container:7cc43e5b4e2a7f2f66f1df774d1a0c63", + "urn": "urn:li:container:7cc43e5b4e2a7f2f66f1df774d1a0c63" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:54727d9fd7deacef27641559125bbc56", @@ -3081,6 +3779,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:54727d9fd7deacef27641559125bbc56", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:141b0980dcb08f48544583e47cf48807", @@ -3164,6 +3882,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:141b0980dcb08f48544583e47cf48807", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:c6627af82d44de89492e1a9315ae9f4b", @@ -3246,5 +3984,25 @@ "lastObserved": 1615443388097, "runId": "mssql-test" } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c6627af82d44de89492e1a9315ae9f4b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json index ad0c3fb818fb2..bc81ce9633432 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json @@ -66,6 +66,21 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", @@ -149,6 +164,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", @@ -232,6 +267,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", @@ -315,6 +370,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", @@ -398,6 +473,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", @@ -481,6 +576,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", @@ -564,6 +679,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", @@ -647,6 +782,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", @@ -730,6 +885,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", @@ -813,6 +988,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", @@ -896,6 +1091,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)", @@ -1000,6 +1215,30 @@ "runId": "mssql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", + "urn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", @@ -1083,6 +1322,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)", @@ -1188,6 +1447,30 @@ "runId": "mssql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)", @@ -1317,6 +1600,30 @@ "runId": "mssql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)", @@ -1433,6 +1740,30 @@ "runId": "mssql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", @@ -1516,6 +1847,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", @@ -1599,6 +1950,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", @@ -1681,5 +2052,25 @@ "lastObserved": 1615443388097, "runId": "mssql-test" } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json index aa00cc9c01b79..8be2fe134dca1 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json @@ -66,6 +66,21 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", @@ -149,6 +164,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", @@ -232,6 +267,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", @@ -315,6 +370,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", @@ -398,6 +473,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", @@ -481,6 +576,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", @@ -564,6 +679,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", @@ -647,6 +782,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", @@ -730,6 +885,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", @@ -813,6 +988,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", @@ -896,6 +1091,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoDataAlias.dbo.Products,PROD)", @@ -1000,6 +1215,30 @@ "runId": "mssql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoDataAlias.dbo.Products,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", + "urn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", @@ -1083,6 +1322,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoDataAlias.Foo.Items,PROD)", @@ -1188,6 +1447,30 @@ "runId": "mssql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoDataAlias.Foo.Items,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoDataAlias.Foo.Persons,PROD)", @@ -1317,6 +1600,30 @@ "runId": "mssql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoDataAlias.Foo.Persons,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoDataAlias.Foo.SalesReason,PROD)", @@ -1433,6 +1740,30 @@ "runId": "mssql-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoDataAlias.Foo.SalesReason,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", @@ -1516,6 +1847,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", @@ -1599,6 +1950,26 @@ "runId": "mssql-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", @@ -1681,5 +2052,25 @@ "lastObserved": 1615443388097, "runId": "mssql-test" } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json index 3521da7dfa4a7..665541a140cc9 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json @@ -65,6 +65,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", @@ -131,6 +146,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", @@ -197,6 +227,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", @@ -278,6 +323,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", @@ -385,6 +450,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", @@ -511,6 +596,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", @@ -618,6 +723,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", @@ -725,6 +850,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", @@ -832,6 +977,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", + "urn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", @@ -1103,6 +1272,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -1640,6 +1833,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -2229,6 +2446,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -2714,6 +2955,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -2835,6 +3100,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -3124,6 +3413,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -3297,6 +3610,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -3774,6 +4111,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -4219,6 +4580,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -4696,6 +5081,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -5054,6 +5463,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -5392,6 +5825,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -5753,6 +6210,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -6169,6 +6650,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -6559,6 +7064,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -6865,6 +7394,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -7203,6 +7756,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -7431,6 +8008,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -7847,6 +8448,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -8159,6 +8784,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -8497,6 +9146,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -8861,6 +9534,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -9277,6 +9974,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -9441,6 +10162,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", @@ -9542,6 +10287,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9620,6 +10389,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9696,6 +10489,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9786,6 +10603,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", @@ -11953,6 +12794,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", @@ -12325,6 +13190,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", @@ -12809,6 +13698,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", @@ -14030,6 +14943,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", @@ -20826,6 +21763,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", @@ -24951,6 +25912,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", @@ -30300,6 +31285,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", @@ -30689,6 +31698,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", @@ -31105,6 +32138,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", @@ -32155,6 +33208,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "urn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", @@ -32354,6 +33427,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", @@ -32558,6 +33655,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "test publish datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -32731,6 +33850,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -32982,6 +34126,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33207,6 +34376,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33380,6 +34574,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33540,6 +34759,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33687,6 +34931,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33782,6 +35051,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33851,6 +35145,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33920,6 +35236,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -34236,6 +35574,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -34245,8 +35605,8 @@ "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ "/prod/tableau/default/Executive Dashboard/Problems", - "/prod/tableau/default/Executive Dashboard/Incidents", - "/prod/tableau/default/Executive Dashboard/Requests" + "/prod/tableau/default/Executive Dashboard/Requests", + "/prod/tableau/default/Executive Dashboard/Incidents" ] } }, @@ -35087,6 +36447,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -36040,6 +37425,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -37071,6 +38481,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -37959,6 +39394,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -38288,6 +39748,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -39202,6 +40687,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -40272,6 +41782,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -41199,6 +42734,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -41220,6 +42780,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -41241,6 +42823,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -41261,6 +42865,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "test publish datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", diff --git a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json index e76eef16766b8..45b1ebca25b8a 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json @@ -65,6 +65,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", @@ -131,6 +146,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", @@ -197,6 +227,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", @@ -304,6 +349,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", @@ -430,6 +495,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", @@ -537,6 +622,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", @@ -644,6 +749,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", @@ -915,6 +1040,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -1452,6 +1601,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -2041,6 +2214,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -2526,6 +2723,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -2647,6 +2868,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -2936,6 +3181,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -3109,6 +3378,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -3586,6 +3879,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -4031,6 +4348,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -4508,6 +4849,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -4866,6 +5231,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -5204,6 +5593,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -5565,6 +5978,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -5981,6 +6418,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -6371,6 +6832,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -6677,6 +7162,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -7015,6 +7524,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -7243,6 +7776,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -7659,6 +8216,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -7971,6 +8552,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -8309,6 +8914,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -8673,6 +9302,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -9089,6 +9742,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -9253,6 +9930,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", @@ -9354,6 +10055,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9432,6 +10157,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9508,6 +10257,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9598,6 +10371,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", @@ -11765,6 +12562,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", @@ -12137,6 +12958,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", @@ -12621,6 +13466,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", @@ -13842,6 +14711,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", @@ -20638,6 +21531,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", @@ -24763,6 +25680,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", @@ -30112,6 +31053,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", @@ -30501,6 +31466,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", @@ -30917,6 +31906,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", @@ -31967,6 +32976,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "urn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", @@ -32166,6 +33195,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", @@ -32370,6 +33423,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "test publish datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -32543,6 +33618,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -32794,6 +33894,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33019,6 +34144,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33192,6 +34342,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33352,6 +34527,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33499,6 +34699,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33594,6 +34819,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33663,6 +34913,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33732,6 +35004,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -34048,6 +35342,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -34057,8 +35373,8 @@ "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ "/prod/tableau/default/Executive Dashboard/Problems", - "/prod/tableau/default/Executive Dashboard/Incidents", - "/prod/tableau/default/Executive Dashboard/Requests" + "/prod/tableau/default/Executive Dashboard/Requests", + "/prod/tableau/default/Executive Dashboard/Incidents" ] } }, @@ -34899,6 +36215,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -35852,6 +37193,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -36883,6 +38249,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -37771,6 +39162,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -38100,6 +39516,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -39014,6 +40455,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -40084,6 +41550,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -41011,6 +42502,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -41032,6 +42548,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -41053,6 +42591,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -41073,6 +42633,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "test publish datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", diff --git a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden_deleted_stateful.json b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden_deleted_stateful.json index 1e9da1d2a3736..96df70dd5b02a 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden_deleted_stateful.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden_deleted_stateful.json @@ -65,6 +65,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", @@ -131,6 +146,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", @@ -198,13 +228,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "browsePathsV2", "aspect": { "json": { - "removed": true + "path": [] } }, "systemMetadata": { @@ -214,7 +244,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -228,8 +258,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "entityType": "tag", + "entityUrn": "urn:li:tag:GROUPFIELD", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -244,7 +274,7 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -259,7 +289,7 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -274,7 +304,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -289,7 +319,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -303,8 +333,8 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:DIMENSION", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -318,8 +348,8 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -333,8 +363,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -348,8 +378,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -363,8 +393,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -378,8 +408,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -393,8 +423,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -408,8 +438,8 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:DATASOURCEFIELD", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -423,8 +453,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -438,8 +468,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "entityType": "tag", + "entityUrn": "urn:li:tag:BINFIELD", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -453,8 +483,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -469,7 +499,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -484,7 +514,7 @@ }, { "entityType": "tag", - "entityUrn": "urn:li:tag:CALCULATEDFIELD", + "entityUrn": "urn:li:tag:TagSheet3", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -498,8 +528,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -513,8 +543,8 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:ATTRIBUTE", + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -528,8 +558,8 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:SUM", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -544,7 +574,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -558,8 +588,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -573,8 +603,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "entityType": "tag", + "entityUrn": "urn:li:tag:COLUMNFIELD", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -588,8 +618,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -604,7 +634,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -619,7 +649,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -633,8 +663,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -649,7 +679,7 @@ }, { "entityType": "tag", - "entityUrn": "urn:li:tag:MEASURE", + "entityUrn": "urn:li:tag:CALCULATEDFIELD", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -663,8 +693,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -679,7 +709,7 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -693,8 +723,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -708,8 +738,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "entityType": "tag", + "entityUrn": "urn:li:tag:SUM", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -723,8 +753,8 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -738,8 +768,8 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -753,8 +783,8 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -769,7 +799,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -798,8 +828,8 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:TagSheet3", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -814,7 +844,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -829,7 +859,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -843,8 +873,8 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -858,8 +888,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "entityType": "tag", + "entityUrn": "urn:li:tag:DATASOURCEFIELD", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -873,8 +903,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -889,7 +919,7 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -904,7 +934,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -918,8 +948,8 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:YEAR", + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -934,7 +964,7 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -948,8 +978,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -964,7 +994,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -979,7 +1009,7 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -994,7 +1024,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1008,8 +1038,8 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:BINFIELD", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1023,8 +1053,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1038,8 +1068,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1054,7 +1084,7 @@ }, { "entityType": "tag", - "entityUrn": "urn:li:tag:GROUPFIELD", + "entityUrn": "urn:li:tag:DIMENSION", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1069,7 +1099,7 @@ }, { "entityType": "tag", - "entityUrn": "urn:li:tag:COUNT", + "entityUrn": "urn:li:tag:YEAR", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1083,8 +1113,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1098,8 +1128,8 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1113,8 +1143,8 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:COLUMNFIELD", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1129,7 +1159,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1143,8 +1173,8 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:SETFIELD", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1159,7 +1189,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1174,7 +1204,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1189,7 +1219,7 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1203,8 +1233,23 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": true + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1219,7 +1264,7 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1234,7 +1279,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1248,8 +1293,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "entityType": "tag", + "entityUrn": "urn:li:tag:COUNT", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1263,8 +1308,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1278,8 +1323,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1294,7 +1339,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1309,7 +1354,7 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1323,8 +1368,8 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "entityType": "tag", + "entityUrn": "urn:li:tag:MEASURE", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1338,8 +1383,8 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1353,8 +1398,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "entityType": "tag", + "entityUrn": "urn:li:tag:SETFIELD", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1368,8 +1413,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "entityType": "tag", + "entityUrn": "urn:li:tag:ATTRIBUTE", "changeType": "UPSERT", "aspectName": "status", "aspect": { diff --git a/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json index 3521da7dfa4a7..665541a140cc9 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json @@ -65,6 +65,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", @@ -131,6 +146,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", @@ -197,6 +227,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", @@ -278,6 +323,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", @@ -385,6 +450,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", @@ -511,6 +596,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", @@ -618,6 +723,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", @@ -725,6 +850,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", @@ -832,6 +977,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", + "urn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", @@ -1103,6 +1272,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -1640,6 +1833,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -2229,6 +2446,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -2714,6 +2955,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -2835,6 +3100,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -3124,6 +3413,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -3297,6 +3610,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -3774,6 +4111,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -4219,6 +4580,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -4696,6 +5081,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -5054,6 +5463,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -5392,6 +5825,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -5753,6 +6210,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -6169,6 +6650,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -6559,6 +7064,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -6865,6 +7394,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -7203,6 +7756,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -7431,6 +8008,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -7847,6 +8448,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -8159,6 +8784,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -8497,6 +9146,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -8861,6 +9534,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -9277,6 +9974,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -9441,6 +10162,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", @@ -9542,6 +10287,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9620,6 +10389,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9696,6 +10489,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9786,6 +10603,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", @@ -11953,6 +12794,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", @@ -12325,6 +13190,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", @@ -12809,6 +13698,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", @@ -14030,6 +14943,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", @@ -20826,6 +21763,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", @@ -24951,6 +25912,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", @@ -30300,6 +31285,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", @@ -30689,6 +31698,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", @@ -31105,6 +32138,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", @@ -32155,6 +33208,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "urn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", @@ -32354,6 +33427,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", @@ -32558,6 +33655,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "test publish datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -32731,6 +33850,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -32982,6 +34126,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33207,6 +34376,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33380,6 +34574,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33540,6 +34759,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33687,6 +34931,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33782,6 +35051,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33851,6 +35145,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33920,6 +35236,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -34236,6 +35574,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -34245,8 +35605,8 @@ "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ "/prod/tableau/default/Executive Dashboard/Problems", - "/prod/tableau/default/Executive Dashboard/Incidents", - "/prod/tableau/default/Executive Dashboard/Requests" + "/prod/tableau/default/Executive Dashboard/Requests", + "/prod/tableau/default/Executive Dashboard/Incidents" ] } }, @@ -35087,6 +36447,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -36040,6 +37425,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -37071,6 +38481,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -37959,6 +39394,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -38288,6 +39748,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -39202,6 +40687,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -40272,6 +41782,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -41199,6 +42734,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -41220,6 +42780,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -41241,6 +42823,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -41261,6 +42865,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "test publish datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", diff --git a/metadata-ingestion/tests/integration/tableau/tableau_project_path_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_project_path_mces_golden.json index b657ae741daef..fb71b8f2c1a57 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_project_path_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_project_path_mces_golden.json @@ -80,6 +80,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", @@ -146,6 +166,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", @@ -252,5 +287,29 @@ "lastObserved": 1638860400000, "runId": "tableau-test" } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", + "urn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json index e76eef16766b8..45b1ebca25b8a 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json @@ -65,6 +65,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", @@ -131,6 +146,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", @@ -197,6 +227,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", @@ -304,6 +349,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", @@ -430,6 +495,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", @@ -537,6 +622,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", @@ -644,6 +749,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", @@ -915,6 +1040,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -1452,6 +1601,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -2041,6 +2214,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -2526,6 +2723,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -2647,6 +2868,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -2936,6 +3181,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -3109,6 +3378,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -3586,6 +3879,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -4031,6 +4348,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -4508,6 +4849,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -4866,6 +5231,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -5204,6 +5593,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -5565,6 +5978,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -5981,6 +6418,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -6371,6 +6832,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -6677,6 +7162,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -7015,6 +7524,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -7243,6 +7776,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -7659,6 +8216,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -7971,6 +8552,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -8309,6 +8914,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -8673,6 +9302,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -9089,6 +9742,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -9253,6 +9930,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", @@ -9354,6 +10055,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9432,6 +10157,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9508,6 +10257,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9598,6 +10371,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", @@ -11765,6 +12562,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", @@ -12137,6 +12958,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", @@ -12621,6 +13466,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", @@ -13842,6 +14711,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", @@ -20638,6 +21531,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", @@ -24763,6 +25680,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", @@ -30112,6 +31053,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", @@ -30501,6 +31466,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", @@ -30917,6 +31906,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", @@ -31967,6 +32976,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "urn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", @@ -32166,6 +33195,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", @@ -32370,6 +33423,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "test publish datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -32543,6 +33618,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -32794,6 +33894,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33019,6 +34144,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33192,6 +34342,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33352,6 +34527,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33499,6 +34699,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33594,6 +34819,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33663,6 +34913,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33732,6 +35004,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -34048,6 +35342,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -34057,8 +35373,8 @@ "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ "/prod/tableau/default/Executive Dashboard/Problems", - "/prod/tableau/default/Executive Dashboard/Incidents", - "/prod/tableau/default/Executive Dashboard/Requests" + "/prod/tableau/default/Executive Dashboard/Requests", + "/prod/tableau/default/Executive Dashboard/Incidents" ] } }, @@ -34899,6 +36215,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -35852,6 +37193,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -36883,6 +38249,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -37771,6 +39162,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -38100,6 +39516,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -39014,6 +40455,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -40084,6 +41550,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -41011,6 +42502,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -41032,6 +42548,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -41053,6 +42591,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -41073,6 +42633,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "test publish datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", diff --git a/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json index 1bff8f293cea2..f82cf56ce2b81 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json @@ -67,6 +67,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:89fbc376e636670d29a60ef2347c156f", @@ -135,6 +150,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:89fbc376e636670d29a60ef2347c156f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5", @@ -244,6 +274,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:fd3437da0a5c1a43130608562ba3f532", @@ -372,6 +422,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fd3437da0a5c1a43130608562ba3f532", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:c209b64f2002cc0839094edcef4b180e", @@ -481,6 +551,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c209b64f2002cc0839094edcef4b180e", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:ba8a5ac7eb4c6e5edc9b03bf8891be55", @@ -590,6 +680,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ba8a5ac7eb4c6e5edc9b03bf8891be55", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(tableau,acryl_site1.222d1406-de0e-cd8d-0b94-9b45a0007e59)", @@ -862,6 +972,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5", + "urn": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -1400,6 +1534,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5", + "urn": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -1990,6 +2148,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5", + "urn": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -2476,119 +2658,21 @@ "runId": "tableau-test" } }, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,acryl_site1.8a6a269a-d6de-fae4-5050-513255b40ffc)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, - "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet1", - "title": "Sheet 1", - "description": "", - "lastModified": { - "created": { - "time": 1639772911000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1642199995000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Dvdrental Workbook" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, { "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.8a6a269a-d6de-fae4-5050-513255b40ffc)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:fd3437da0a5c1a43130608562ba3f532" - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,acryl_site1.8a6a269a-d6de-fae4-5050-513255b40ffc)", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", "changeType": "UPSERT", - "aspectName": "inputFields", + "aspectName": "browsePathsV2", "aspect": { "json": { - "fields": [ + "path": [ { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)", - "schemaField": { - "fieldPath": "Custom SQL Query", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.schema.ArrayType": {} - } - }, - "nativeDataType": "TABLE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - } + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5", + "urn": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5" } ] } @@ -2601,7 +2685,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,acryl_site1.c57a5574-db47-46df-677f-0b708dab14db)", + "urn": "urn:li:chart:(tableau,acryl_site1.8a6a269a-d6de-fae4-5050-513255b40ffc)", "aspects": [ { "com.linkedin.pegasus2avro.common.DataPlatformInstance": { @@ -2614,12 +2698,158 @@ "customProperties": { "luid": "" }, - "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet2", - "title": "Sheet 2", + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet1", + "title": "Sheet 1", "description": "", "lastModified": { "created": { - "time": 1639773415000, + "time": 1639772911000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fd3437da0a5c1a43130608562ba3f532" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)", + "schemaField": { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:fd3437da0a5c1a43130608562ba3f532", + "urn": "urn:li:container:fd3437da0a5c1a43130608562ba3f532" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,acryl_site1.c57a5574-db47-46df-677f-0b708dab14db)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "luid": "" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet2", + "title": "Sheet 2", + "description": "", + "lastModified": { + "created": { + "time": 1639773415000, "actor": "urn:li:corpuser:jawadqu@gmail.com" }, "lastModified": { @@ -2888,6 +3118,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:fd3437da0a5c1a43130608562ba3f532", + "urn": "urn:li:container:fd3437da0a5c1a43130608562ba3f532" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -3062,6 +3316,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:fd3437da0a5c1a43130608562ba3f532", + "urn": "urn:li:container:fd3437da0a5c1a43130608562ba3f532" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -3540,6 +3818,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", + "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -3986,6 +4288,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", + "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -4464,6 +4790,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", + "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -4823,6 +5173,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", + "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -5162,6 +5536,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", + "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -5524,6 +5922,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", + "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -5941,6 +6363,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", + "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -6332,6 +6778,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", + "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -6639,6 +7109,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", + "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -6978,6 +7472,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", + "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -7207,6 +7725,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", + "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -7624,6 +8166,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", + "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -7937,6 +8503,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", + "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -8276,6 +8866,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", + "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -8641,6 +9255,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", + "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -9058,6 +9696,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", + "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { @@ -9223,6 +9885,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,acryl_site1.130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:ba8a5ac7eb4c6e5edc9b03bf8891be55", + "urn": "urn:li:container:ba8a5ac7eb4c6e5edc9b03bf8891be55" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(tableau,acryl_site1.8f7dd564-36b6-593f-3c6f-687ad06cd40b)", @@ -9325,6 +10011,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,acryl_site1.8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5", + "urn": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9404,6 +10114,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,acryl_site1.20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:fd3437da0a5c1a43130608562ba3f532", + "urn": "urn:li:container:fd3437da0a5c1a43130608562ba3f532" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9481,6 +10215,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,acryl_site1.39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:fd3437da0a5c1a43130608562ba3f532", + "urn": "urn:li:container:fd3437da0a5c1a43130608562ba3f532" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { @@ -9572,6 +10330,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,acryl_site1.5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", + "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", @@ -11740,6 +12522,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5", + "urn": "urn:li:container:8533f62e6c2cd132ff69f1ff5a9ce1f5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", @@ -12113,6 +12919,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:fd3437da0a5c1a43130608562ba3f532", + "urn": "urn:li:container:fd3437da0a5c1a43130608562ba3f532" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", @@ -12598,6 +13428,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:fd3437da0a5c1a43130608562ba3f532", + "urn": "urn:li:container:fd3437da0a5c1a43130608562ba3f532" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", @@ -13820,6 +14674,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:fd3437da0a5c1a43130608562ba3f532", + "urn": "urn:li:container:fd3437da0a5c1a43130608562ba3f532" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", @@ -20617,6 +21495,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", + "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", @@ -24743,6 +25645,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", + "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", @@ -30093,6 +31019,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:c209b64f2002cc0839094edcef4b180e", + "urn": "urn:li:container:c209b64f2002cc0839094edcef4b180e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", @@ -30483,6 +31433,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:ba8a5ac7eb4c6e5edc9b03bf8891be55", + "urn": "urn:li:container:ba8a5ac7eb4c6e5edc9b03bf8891be55" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", @@ -30900,6 +31874,26 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", @@ -31936,6 +32930,25 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", @@ -32136,6 +33149,30 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", + "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" + }, + { + "id": "urn:li:container:fd3437da0a5c1a43130608562ba3f532", + "urn": "urn:li:container:fd3437da0a5c1a43130608562ba3f532" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", @@ -32341,6 +33378,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "test publish datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -32514,6 +33573,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -32765,6 +33849,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -32990,6 +34099,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33163,6 +34297,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33323,6 +34482,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33470,6 +34654,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33565,6 +34774,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33634,6 +34868,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -33703,6 +34959,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -34019,6 +35297,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -34028,8 +35328,8 @@ "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ "/prod/tableau/default/Executive Dashboard/Problems", - "/prod/tableau/default/Executive Dashboard/Incidents", - "/prod/tableau/default/Executive Dashboard/Requests" + "/prod/tableau/default/Executive Dashboard/Requests", + "/prod/tableau/default/Executive Dashboard/Incidents" ] } }, @@ -34870,6 +36170,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -35823,6 +37148,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -36854,6 +38204,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -37742,6 +39117,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -38071,6 +39471,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -38985,6 +40410,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -40055,6 +41505,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -40982,6 +42457,31 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -41003,6 +42503,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -41024,6 +42546,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -41044,6 +42588,28 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "test publish datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(tableau,acryl_site1.130496dc-29ca-8a89-e32b-d73c4d8b65ff)", diff --git a/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json b/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json index 319e73bcaca1e..19961e48b4a33 100644 --- a/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json +++ b/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json @@ -66,6 +66,21 @@ "runId": "trino-hive-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", @@ -149,6 +164,26 @@ "runId": "trino-hive-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7", + "urn": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test,PROD)", @@ -184,7 +219,7 @@ "numrows": "1", "rawdatasize": "32", "totalsize": "33", - "transient_lastddltime": "1683850708" + "transient_lastddltime": "1688422059" }, "name": "array_struct_test", "description": "This table has array of structs", @@ -300,6 +335,30 @@ "runId": "trino-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7", + "urn": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7" + }, + { + "id": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", + "urn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.map_test,PROD)", @@ -333,7 +392,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1683850711" + "transient_lastddltime": "1688422063" }, "name": "map_test", "tags": [] @@ -415,6 +474,30 @@ "runId": "trino-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7", + "urn": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7" + }, + { + "id": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", + "urn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.nested_struct_test,PROD)", @@ -448,7 +531,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1683850711" + "transient_lastddltime": "1688422062" }, "name": "nested_struct_test", "tags": [] @@ -579,6 +662,30 @@ "runId": "trino-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7", + "urn": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7" + }, + { + "id": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", + "urn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.pokes,PROD)", @@ -607,7 +714,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1683850703" + "transient_lastddltime": "1688421792" }, "name": "pokes", "tags": [] @@ -697,6 +804,30 @@ "runId": "trino-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7", + "urn": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7" + }, + { + "id": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", + "urn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.struct_test,PROD)", @@ -730,7 +861,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1683850705" + "transient_lastddltime": "1688421808" }, "name": "struct_test", "tags": [] @@ -839,6 +970,30 @@ "runId": "trino-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7", + "urn": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7" + }, + { + "id": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", + "urn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.struct_test_view_materialized,PROD)", @@ -869,7 +1024,7 @@ "customProperties": { "numfiles": "0", "totalsize": "0", - "transient_lastddltime": "1683850711" + "transient_lastddltime": "1688422062" }, "name": "struct_test_view_materialized", "tags": [] @@ -978,6 +1133,30 @@ "runId": "trino-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.struct_test_view_materialized,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7", + "urn": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7" + }, + { + "id": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", + "urn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1._test_table_underscore,PROD)", @@ -1011,7 +1190,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1683850705" + "transient_lastddltime": "1688421807" }, "name": "_test_table_underscore", "tags": [] @@ -1089,6 +1268,30 @@ "runId": "trino-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7", + "urn": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7" + }, + { + "id": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", + "urn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.union_test,PROD)", @@ -1122,7 +1325,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1683850711" + "transient_lastddltime": "1688422062" }, "name": "union_test", "tags": [] @@ -1284,6 +1487,30 @@ "runId": "trino-hive-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7", + "urn": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7" + }, + { + "id": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", + "urn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test_view,PROD)", @@ -1312,7 +1539,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1683850711", + "transient_lastddltime": "1688422062", "view_definition": "SELECT \"property_id\", \"service\"\nFROM \"db1\".\"array_struct_test\"", "is_view": "True" }, @@ -1443,5 +1670,29 @@ "lastObserved": 1632398400000, "runId": "trino-hive-test" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,hivedb.db1.array_struct_test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7", + "urn": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7" + }, + { + "id": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", + "urn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-test" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/trino/trino_mces_golden.json b/metadata-ingestion/tests/integration/trino/trino_mces_golden.json index 3e7f2f1a61a8d..dce10ce377be3 100644 --- a/metadata-ingestion/tests/integration/trino/trino_mces_golden.json +++ b/metadata-ingestion/tests/integration/trino/trino_mces_golden.json @@ -66,6 +66,21 @@ "runId": "trino-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:57aba13b10c1691508600999cd411c25", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:8568c0971ce28c183fde0dff7f88e617", @@ -149,6 +164,26 @@ "runId": "trino-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:8568c0971ce28c183fde0dff7f88e617", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:57aba13b10c1691508600999cd411c25", + "urn": "urn:li:container:57aba13b10c1691508600999cd411c25" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,library_catalog.librarydb.book,PROD)", @@ -306,6 +341,30 @@ "runId": "trino-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,library_catalog.librarydb.book,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:57aba13b10c1691508600999cd411c25", + "urn": "urn:li:container:57aba13b10c1691508600999cd411c25" + }, + { + "id": "urn:li:container:8568c0971ce28c183fde0dff7f88e617", + "urn": "urn:li:container:8568c0971ce28c183fde0dff7f88e617" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,library_catalog.librarydb.issue_history,PROD)", @@ -434,6 +493,30 @@ "runId": "trino-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,library_catalog.librarydb.issue_history,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:57aba13b10c1691508600999cd411c25", + "urn": "urn:li:container:57aba13b10c1691508600999cd411c25" + }, + { + "id": "urn:li:container:8568c0971ce28c183fde0dff7f88e617", + "urn": "urn:li:container:8568c0971ce28c183fde0dff7f88e617" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,library_catalog.librarydb.member,PROD)", @@ -538,6 +621,30 @@ "runId": "trino-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,library_catalog.librarydb.member,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:57aba13b10c1691508600999cd411c25", + "urn": "urn:li:container:57aba13b10c1691508600999cd411c25" + }, + { + "id": "urn:li:container:8568c0971ce28c183fde0dff7f88e617", + "urn": "urn:li:container:8568c0971ce28c183fde0dff7f88e617" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,library_catalog.librarydb.book_in_circulation,PROD)", @@ -690,6 +797,30 @@ "runId": "trino-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,library_catalog.librarydb.book_in_circulation,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:57aba13b10c1691508600999cd411c25", + "urn": "urn:li:container:57aba13b10c1691508600999cd411c25" + }, + { + "id": "urn:li:container:8568c0971ce28c183fde0dff7f88e617", + "urn": "urn:li:container:8568c0971ce28c183fde0dff7f88e617" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,library_catalog.librarydb.book,PROD)", diff --git a/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json b/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json index f37170bdae547..0c14096345d7e 100644 --- a/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json +++ b/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json @@ -90,6 +90,21 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:83d98e62e36bddc3596c2b738e23b596", @@ -198,6 +213,26 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:83d98e62e36bddc3596c2b738e23b596", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "urn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:5ada0a9773235325e506410c512feabb", @@ -307,6 +342,30 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ada0a9773235325e506410c512feabb", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "urn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb" + }, + { + "id": "urn:li:container:83d98e62e36bddc3596c2b738e23b596", + "urn": "urn:li:container:83d98e62e36bddc3596c2b738e23b596" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.default.quickstart_table,PROD)", @@ -484,6 +543,34 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.default.quickstart_table,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "urn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb" + }, + { + "id": "urn:li:container:83d98e62e36bddc3596c2b738e23b596", + "urn": "urn:li:container:83d98e62e36bddc3596c2b738e23b596" + }, + { + "id": "urn:li:container:5ada0a9773235325e506410c512feabb", + "urn": "urn:li:container:5ada0a9773235325e506410c512feabb" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:0e09e6ec299ef004941e25221d3ef6b6", @@ -593,6 +680,30 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0e09e6ec299ef004941e25221d3ef6b6", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "urn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb" + }, + { + "id": "urn:li:container:83d98e62e36bddc3596c2b738e23b596", + "urn": "urn:li:container:83d98e62e36bddc3596c2b738e23b596" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.information_schema.quickstart_table,PROD)", @@ -770,6 +881,34 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.information_schema.quickstart_table,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "urn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb" + }, + { + "id": "urn:li:container:83d98e62e36bddc3596c2b738e23b596", + "urn": "urn:li:container:83d98e62e36bddc3596c2b738e23b596" + }, + { + "id": "urn:li:container:0e09e6ec299ef004941e25221d3ef6b6", + "urn": "urn:li:container:0e09e6ec299ef004941e25221d3ef6b6" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:481380c5a355638fc626eca8380cdda9", @@ -879,6 +1018,30 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:481380c5a355638fc626eca8380cdda9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "urn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb" + }, + { + "id": "urn:li:container:83d98e62e36bddc3596c2b738e23b596", + "urn": "urn:li:container:83d98e62e36bddc3596c2b738e23b596" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.quickstart_schema.quickstart_table,PROD)", @@ -1056,6 +1219,34 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.quickstart_schema.quickstart_table,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "urn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb" + }, + { + "id": "urn:li:container:83d98e62e36bddc3596c2b738e23b596", + "urn": "urn:li:container:83d98e62e36bddc3596c2b738e23b596" + }, + { + "id": "urn:li:container:481380c5a355638fc626eca8380cdda9", + "urn": "urn:li:container:481380c5a355638fc626eca8380cdda9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:079ede9d4f0640985a8ccf8eb180e965", @@ -1164,6 +1355,26 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:079ede9d4f0640985a8ccf8eb180e965", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "urn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:ce568b660cba2e1a4e811b010ac27f90", @@ -1273,6 +1484,30 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ce568b660cba2e1a4e811b010ac27f90", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "urn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb" + }, + { + "id": "urn:li:container:079ede9d4f0640985a8ccf8eb180e965", + "urn": "urn:li:container:079ede9d4f0640985a8ccf8eb180e965" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.default.quickstart_table,PROD)", @@ -1450,6 +1685,34 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.default.quickstart_table,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "urn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb" + }, + { + "id": "urn:li:container:079ede9d4f0640985a8ccf8eb180e965", + "urn": "urn:li:container:079ede9d4f0640985a8ccf8eb180e965" + }, + { + "id": "urn:li:container:ce568b660cba2e1a4e811b010ac27f90", + "urn": "urn:li:container:ce568b660cba2e1a4e811b010ac27f90" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:29f99476d533719be0cebc374d5265dc", @@ -1559,6 +1822,30 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:29f99476d533719be0cebc374d5265dc", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "urn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb" + }, + { + "id": "urn:li:container:079ede9d4f0640985a8ccf8eb180e965", + "urn": "urn:li:container:079ede9d4f0640985a8ccf8eb180e965" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.information_schema.quickstart_table,PROD)", @@ -1736,6 +2023,34 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.information_schema.quickstart_table,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "urn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb" + }, + { + "id": "urn:li:container:079ede9d4f0640985a8ccf8eb180e965", + "urn": "urn:li:container:079ede9d4f0640985a8ccf8eb180e965" + }, + { + "id": "urn:li:container:29f99476d533719be0cebc374d5265dc", + "urn": "urn:li:container:29f99476d533719be0cebc374d5265dc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:47a033e31b92a120f08f297c05d286f1", @@ -1845,6 +2160,30 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:47a033e31b92a120f08f297c05d286f1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "urn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb" + }, + { + "id": "urn:li:container:079ede9d4f0640985a8ccf8eb180e965", + "urn": "urn:li:container:079ede9d4f0640985a8ccf8eb180e965" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.quickstart_schema.quickstart_table,PROD)", @@ -2022,6 +2361,34 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.quickstart_catalog.quickstart_schema.quickstart_table,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "urn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb" + }, + { + "id": "urn:li:container:079ede9d4f0640985a8ccf8eb180e965", + "urn": "urn:li:container:079ede9d4f0640985a8ccf8eb180e965" + }, + { + "id": "urn:li:container:47a033e31b92a120f08f297c05d286f1", + "urn": "urn:li:container:47a033e31b92a120f08f297c05d286f1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:5f7e6ee26826ba56e6d1d0b94f291fa9", @@ -2130,6 +2497,26 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5f7e6ee26826ba56e6d1d0b94f291fa9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "urn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:b330768923270ff5450695bee1c94247", @@ -2239,6 +2626,30 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:b330768923270ff5450695bee1c94247", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "urn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb" + }, + { + "id": "urn:li:container:5f7e6ee26826ba56e6d1d0b94f291fa9", + "urn": "urn:li:container:5f7e6ee26826ba56e6d1d0b94f291fa9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.default.quickstart_table,PROD)", @@ -2416,6 +2827,34 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.default.quickstart_table,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "urn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb" + }, + { + "id": "urn:li:container:5f7e6ee26826ba56e6d1d0b94f291fa9", + "urn": "urn:li:container:5f7e6ee26826ba56e6d1d0b94f291fa9" + }, + { + "id": "urn:li:container:b330768923270ff5450695bee1c94247", + "urn": "urn:li:container:b330768923270ff5450695bee1c94247" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:cb26af5fb7ba2e1c6f2cd804101a5a59", @@ -2525,6 +2964,30 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:cb26af5fb7ba2e1c6f2cd804101a5a59", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "urn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb" + }, + { + "id": "urn:li:container:5f7e6ee26826ba56e6d1d0b94f291fa9", + "urn": "urn:li:container:5f7e6ee26826ba56e6d1d0b94f291fa9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.information_schema.quickstart_table,PROD)", @@ -2702,6 +3165,34 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.information_schema.quickstart_table,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "urn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb" + }, + { + "id": "urn:li:container:5f7e6ee26826ba56e6d1d0b94f291fa9", + "urn": "urn:li:container:5f7e6ee26826ba56e6d1d0b94f291fa9" + }, + { + "id": "urn:li:container:cb26af5fb7ba2e1c6f2cd804101a5a59", + "urn": "urn:li:container:cb26af5fb7ba2e1c6f2cd804101a5a59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:bd1cfbc8ea4c94a6f8c9b1ddba52a8c1", @@ -2811,6 +3302,30 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bd1cfbc8ea4c94a6f8c9b1ddba52a8c1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "urn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb" + }, + { + "id": "urn:li:container:5f7e6ee26826ba56e6d1d0b94f291fa9", + "urn": "urn:li:container:5f7e6ee26826ba56e6d1d0b94f291fa9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.quickstart_schema.quickstart_table,PROD)", @@ -2988,6 +3503,34 @@ "runId": "unity-catalog-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.system.quickstart_schema.quickstart_table,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb", + "urn": "urn:li:container:22ec33be0e53ba3e61bb6c4ad58f6ffb" + }, + { + "id": "urn:li:container:5f7e6ee26826ba56e6d1d0b94f291fa9", + "urn": "urn:li:container:5f7e6ee26826ba56e6d1d0b94f291fa9" + }, + { + "id": "urn:li:container:bd1cfbc8ea4c94a6f8c9b1ddba52a8c1", + "urn": "urn:li:container:bd1cfbc8ea4c94a6f8c9b1ddba52a8c1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,dummy.acryl_metastore.main.default.quickstart_table,PROD)", diff --git a/metadata-ingestion/tests/unit/glue/glue_deleted_actor_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_deleted_actor_mces_golden.json index abf36c78dc72f..51d5a6974eb0d 100644 --- a/metadata-ingestion/tests/unit/glue/glue_deleted_actor_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_deleted_actor_mces_golden.json @@ -67,6 +67,21 @@ "runId": "glue-2020_04_14-07_00_00" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "glue-2020_04_14-07_00_00" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -251,6 +266,26 @@ "runId": "glue-2020_04_14-07_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_jsons_markers,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", + "urn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "glue-2020_04_14-07_00_00" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -437,13 +472,18 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "browsePathsV2", "aspect": { "json": { - "removed": true + "path": [ + { + "id": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", + "urn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7" + } + ] } }, "systemMetadata": { @@ -465,5 +505,20 @@ "lastObserved": 1586847600000, "runId": "glue-2020_04_14-07_00_00" } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": true + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "glue-2020_04_14-07_00_00" + } } ] \ No newline at end of file From 4daee108c203e4efe6669937e62999af2308711b Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 6 Jul 2023 13:53:03 -0700 Subject: [PATCH 108/222] chore(ingest/delta-lake): cleanup import error handling (#8230) Co-authored-by: Tamas Nemeth --- .../source/delta_lake/delta_lake_utils.py | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/delta_lake_utils.py b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/delta_lake_utils.py index 15b1c9006f9a9..071dacae4a8c5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/delta_lake_utils.py +++ b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/delta_lake_utils.py @@ -6,15 +6,12 @@ try: from deltalake.exceptions import TableNotFoundError - DELTALAKE_VERSION_GTE_0_10_0 = True - - + _MUST_CHECK_TABLE_NOT_FOUND_MESSAGE = False except ImportError: - # For deltalake < 0.10.0 - from deltalake import PyDeltaTableError # type: ignore[attr-defined] - - DELTALAKE_VERSION_GTE_0_10_0 = False + # For deltalake < 0.10.0. + from deltalake import PyDeltaTableError as TableNotFoundError # type: ignore + _MUST_CHECK_TABLE_NOT_FOUND_MESSAGE = True from datahub.ingestion.source.delta_lake.config import DeltaLakeSourceConfig @@ -33,15 +30,15 @@ def read_delta_table( storage_options=opts, without_files=not delta_lake_config.require_files, ) - except Exception as e: - if (DELTALAKE_VERSION_GTE_0_10_0 and isinstance(e, TableNotFoundError)) or ( - not DELTALAKE_VERSION_GTE_0_10_0 - and isinstance(e, PyDeltaTableError) - and "Not a Delta table" in str(e) - ): - pass - else: + except TableNotFoundError as e: + # For deltalake < 0.10.0, we need to check the error message to make sure + # that this is a table not found error. Newer versions have a dedicated + # exception class for this. + if _MUST_CHECK_TABLE_NOT_FOUND_MESSAGE and "Not a Delta table" not in str(e): raise e + else: + # Otherwise, the table was genuinely not found and we return None. + pass return None From 74301ecb60bcd56d3e195d83aff7a8ffaedba860 Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Fri, 7 Jul 2023 12:43:01 -0400 Subject: [PATCH 109/222] test(ingest/nifi): Configure nifi tests for arm64 (#8363) Co-authored-by: Mayuri N Co-authored-by: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> --- .../tests/integration/nifi/docker-compose.yml | 11 +- .../nifi/nifi_mces_golden_cluster.json | 587 +++++++++--------- .../nifi/nifi_mces_golden_standalone.json | 196 +++--- .../nifi/setup/conf_clustered/flow.xml.gz | Bin 61643 -> 4673 bytes .../tests/integration/nifi/test_nifi.py | 186 +++--- 5 files changed, 535 insertions(+), 445 deletions(-) diff --git a/metadata-ingestion/tests/integration/nifi/docker-compose.yml b/metadata-ingestion/tests/integration/nifi/docker-compose.yml index fc32865e0c37b..9992a81df57cb 100644 --- a/metadata-ingestion/tests/integration/nifi/docker-compose.yml +++ b/metadata-ingestion/tests/integration/nifi/docker-compose.yml @@ -1,6 +1,6 @@ services: nifi1: - image: apache/nifi:1.15.0 + image: apache/nifi:1.22.0 container_name: nifi1 hostname: nifi1 environment: @@ -48,7 +48,7 @@ services: - "52181" nifi01: - image: apache/nifi:1.15.0 + image: apache/nifi:1.22.0 container_name: nifi01 hostname: nifi01 depends_on: @@ -80,7 +80,7 @@ services: nifi02: - image: apache/nifi:1.15.0 + image: apache/nifi:1.22.0 container_name: nifi02 hostname: nifi02 depends_on: @@ -98,7 +98,7 @@ services: - NIFI_SENSITIVE_PROPS_KEY=admin@datahub nifi03: - image: apache/nifi:1.15.0 + image: apache/nifi:1.22.0 container_name: nifi03 hostname: nifi03 depends_on: @@ -117,10 +117,11 @@ services: sftp_public_host: image: atmoz/sftp + platform: linux/amd64 container_name: sftp_public_host hostname: sftp_public_host volumes: - ./setup/sftp_files:/home/foo ports: - "2222:22" - command: "foo:pass:::" \ No newline at end of file + command: "foo:pass:::" diff --git a/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_cluster.json b/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_cluster.json index 74b446f90f4e5..c38b134a97243 100644 --- a/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_cluster.json +++ b/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_cluster.json @@ -1,629 +1,656 @@ [ { - "auditHeader": null, "entityType": "dataFlow", "entityUrn": "urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataFlowInfo", "aspect": { - "value": "{\"customProperties\": {\"clustered\": \"True\", \"version\": \"1.15.0\"}, \"externalUrl\": \"http://localhost:9080/nifi/?processGroupId=80820b2f-017d-1000-85cf-05f56cde9185&componentIds=\", \"name\": \"Cluster Flow\"}", - "contentType": "application/json" + "json": { + "customProperties": { + "clustered": "True", + "version": "1.22.0" + }, + "externalUrl": "http://localhost:9080/nifi/?processGroupId=80820b2f-017d-1000-85cf-05f56cde9185&componentIds=", + "name": "Cluster Flow" + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),fed5914b-937b-37dd-89c0-b34ffbae9cf4)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataJobInfo", "aspect": { - "value": "{\"customProperties\": {\"schedulingPeriod\": \"0 sec\", \"schedulingStrategy\": \"TIMER_DRIVEN\", \"executionNode\": \"ALL\", \"concurrentlySchedulableTaskCount\": \"1\", \"properties\": \"{\\\"Object Key\\\": \\\"tropical_data/${filename}\\\", \\\"Bucket\\\": \\\"${s3.destbucket}\\\", \\\"Content Type\\\": null, \\\"Content Disposition\\\": null, \\\"Cache Control\\\": null, \\\"Access Key\\\": null, \\\"Secret Key\\\": null, \\\"Credentials File\\\": null, \\\"AWS Credentials Provider service\\\": \\\"69fc8d86-2f01-3f07-910f-2b14edc81779\\\", \\\"s3-object-tags-prefix\\\": null, \\\"s3-object-remove-tags-prefix\\\": \\\"false\\\", \\\"Storage Class\\\": \\\"Standard\\\", \\\"Region\\\": \\\"us-east-1\\\", \\\"Communications Timeout\\\": \\\"30 secs\\\", \\\"Expiration Time Rule\\\": null, \\\"FullControl User List\\\": \\\"${s3.permissions.full.users}\\\", \\\"Read Permission User List\\\": \\\"${s3.permissions.read.users}\\\", \\\"Write Permission User List\\\": \\\"${s3.permissions.write.users}\\\", \\\"Read ACL User List\\\": \\\"${s3.permissions.readacl.users}\\\", \\\"Write ACL User List\\\": \\\"${s3.permissions.writeacl.users}\\\", \\\"Owner\\\": \\\"${s3.owner}\\\", \\\"canned-acl\\\": \\\"${s3.permissions.cannedacl}\\\", \\\"SSL Context Service\\\": null, \\\"Endpoint Override URL\\\": null, \\\"Signer Override\\\": \\\"Default Signature\\\", \\\"Multipart Threshold\\\": \\\"5 GB\\\", \\\"Multipart Part Size\\\": \\\"5 GB\\\", \\\"Multipart Upload AgeOff Interval\\\": \\\"60 min\\\", \\\"Multipart Upload Max Age Threshold\\\": \\\"7 days\\\", \\\"s3-temporary-directory-multipart\\\": \\\"${java.io.tmpdir}\\\", \\\"server-side-encryption\\\": \\\"None\\\", \\\"encryption-service\\\": null, \\\"use-chunked-encoding\\\": \\\"true\\\", \\\"use-path-style-access\\\": \\\"false\\\", \\\"proxy-configuration-service\\\": null, \\\"Proxy Host\\\": null, \\\"Proxy Host Port\\\": null, \\\"proxy-user-name\\\": null, \\\"proxy-user-password\\\": null}\", \"last_event_time\": \"None\"}, \"externalUrl\": \"http://localhost:9080/nifi/?processGroupId=1c61a8d9-3462-387f-8145-09e6e7785e5c&componentIds=fed5914b-937b-37dd-89c0-b34ffbae9cf4\", \"name\": \"PutS3Object\", \"description\": \"\", \"type\": {\"string\": \"NIFI_PROCESSOR\"}}", - "contentType": "application/json" + "json": { + "customProperties": { + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "concurrentlySchedulableTaskCount": "1", + "properties": "{\"Object Key\": \"tropical_data/${filename}\", \"Bucket\": \"${s3.destbucket}\", \"Content Type\": null, \"Content Disposition\": null, \"Cache Control\": null, \"Access Key\": null, \"Secret Key\": null, \"Credentials File\": null, \"AWS Credentials Provider service\": \"69fc8d86-2f01-3f07-910f-2b14edc81779\", \"s3-object-tags-prefix\": null, \"s3-object-remove-tags-prefix\": \"false\", \"Storage Class\": \"Standard\", \"Region\": \"us-east-1\", \"Communications Timeout\": \"30 secs\", \"Expiration Time Rule\": null, \"FullControl User List\": \"${s3.permissions.full.users}\", \"Read Permission User List\": \"${s3.permissions.read.users}\", \"Write Permission User List\": \"${s3.permissions.write.users}\", \"Read ACL User List\": \"${s3.permissions.readacl.users}\", \"Write ACL User List\": \"${s3.permissions.writeacl.users}\", \"Owner\": \"${s3.owner}\", \"canned-acl\": \"${s3.permissions.cannedacl}\", \"SSL Context Service\": null, \"Endpoint Override URL\": null, \"Signer Override\": \"Default Signature\", \"custom-signer-class-name\": null, \"custom-signer-module-location\": null, \"Multipart Threshold\": \"5 GB\", \"Multipart Part Size\": \"5 GB\", \"Multipart Upload AgeOff Interval\": \"60 min\", \"Multipart Upload Max Age Threshold\": \"7 days\", \"s3-temporary-directory-multipart\": \"${java.io.tmpdir}\", \"server-side-encryption\": \"None\", \"encryption-service\": null, \"use-chunked-encoding\": \"true\", \"use-path-style-access\": \"false\", \"proxy-configuration-service\": null, \"Proxy Host\": null, \"Proxy Host Port\": null, \"proxy-user-name\": null, \"proxy-user-password\": null}", + "last_event_time": "None" + }, + "externalUrl": "http://localhost:9080/nifi/?processGroupId=1c61a8d9-3462-387f-8145-09e6e7785e5c&componentIds=fed5914b-937b-37dd-89c0-b34ffbae9cf4", + "name": "PutS3Object", + "description": "", + "type": { + "string": "NIFI_PROCESSOR" + } + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),fed5914b-937b-37dd-89c0-b34ffbae9cf4)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataJobInputOutput", "aspect": { - "value": "{\"inputDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:nifi,default.s3_data,PROD)\"], \"outputDatasets\": [], \"inputDatajobs\": []}", - "contentType": "application/json" + "json": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:nifi,default.s3_data,PROD)" + ], + "outputDatasets": [], + "inputDatajobs": [] + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", - "contentType": "application/json" + "json": { + "platform": "urn:li:dataPlatform:s3" + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { - "value": "{\"customProperties\": {\"s3_uri\": \"s3://enriched-topical-chat\"}, \"tags\": []}", - "contentType": "application/json" + "json": { + "customProperties": { + "s3_uri": "s3://enriched-topical-chat" + }, + "tags": [] + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),c5f6fc66-ffbb-3f60-9564-f2466ae32493)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataJobInfo", "aspect": { - "value": "{\"customProperties\": {\"schedulingPeriod\": \"0 sec\", \"schedulingStrategy\": \"TIMER_DRIVEN\", \"executionNode\": \"ALL\", \"concurrentlySchedulableTaskCount\": \"1\", \"properties\": \"{\\\"Bucket\\\": \\\"enriched-topical-chat\\\", \\\"Object Key\\\": \\\"${filename}\\\", \\\"Region\\\": \\\"us-west-2\\\", \\\"Access Key\\\": null, \\\"Secret Key\\\": null, \\\"Credentials File\\\": null, \\\"AWS Credentials Provider service\\\": null, \\\"Communications Timeout\\\": \\\"30 secs\\\", \\\"Version\\\": null, \\\"SSL Context Service\\\": null, \\\"Endpoint Override URL\\\": null, \\\"Signer Override\\\": \\\"Default Signature\\\", \\\"encryption-service\\\": null, \\\"proxy-configuration-service\\\": null, \\\"Proxy Host\\\": null, \\\"Proxy Host Port\\\": null, \\\"proxy-user-name\\\": null, \\\"proxy-user-password\\\": null, \\\"requester-pays\\\": \\\"false\\\", \\\"range-start\\\": null, \\\"range-length\\\": null}\", \"last_event_time\": \"2022-08-23 19:44:45.343000+00:00\"}, \"externalUrl\": \"http://localhost:9080/nifi/?processGroupId=1c61a8d9-3462-387f-8145-09e6e7785e5c&componentIds=c5f6fc66-ffbb-3f60-9564-f2466ae32493\", \"name\": \"FetchS3Object\", \"description\": \"\", \"type\": {\"string\": \"NIFI_PROCESSOR\"}}", - "contentType": "application/json" + "json": { + "customProperties": { + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "concurrentlySchedulableTaskCount": "1", + "properties": "{\"Bucket\": \"enriched-topical-chat\", \"Object Key\": \"${filename}\", \"Region\": \"us-west-2\", \"Access Key\": null, \"Secret Key\": null, \"Credentials File\": null, \"AWS Credentials Provider service\": null, \"Communications Timeout\": \"30 secs\", \"Version\": null, \"SSL Context Service\": null, \"Endpoint Override URL\": null, \"Signer Override\": \"Default Signature\", \"custom-signer-class-name\": null, \"custom-signer-module-location\": null, \"encryption-service\": null, \"proxy-configuration-service\": null, \"Proxy Host\": null, \"Proxy Host Port\": null, \"proxy-user-name\": null, \"proxy-user-password\": null, \"requester-pays\": \"false\", \"range-start\": null, \"range-length\": null}", + "last_event_time": "2023-07-05 16:10:36.941000+00:00" + }, + "externalUrl": "http://localhost:9080/nifi/?processGroupId=1c61a8d9-3462-387f-8145-09e6e7785e5c&componentIds=c5f6fc66-ffbb-3f60-9564-f2466ae32493", + "name": "FetchS3Object", + "description": "", + "type": { + "string": "NIFI_PROCESSOR" + } + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),c5f6fc66-ffbb-3f60-9564-f2466ae32493)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataJobInputOutput", "aspect": { - "value": "{\"inputDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)\"], \"outputDatasets\": [], \"inputDatajobs\": [\"urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),8a218b6e-e6a0-36b6-bc4b-79d202a80167)\"]}", - "contentType": "application/json" + "json": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)" + ], + "outputDatasets": [], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),8a218b6e-e6a0-36b6-bc4b-79d202a80167)" + ] + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", - "contentType": "application/json" + "json": { + "platform": "urn:li:dataPlatform:s3" + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { - "value": "{\"customProperties\": {\"s3_uri\": \"s3://enriched-topical-chat\"}, \"tags\": []}", - "contentType": "application/json" + "json": { + "customProperties": { + "s3_uri": "s3://enriched-topical-chat" + }, + "tags": [] + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),8a218b6e-e6a0-36b6-bc4b-79d202a80167)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataJobInfo", "aspect": { - "value": "{\"customProperties\": {\"schedulingPeriod\": \"0 sec\", \"schedulingStrategy\": \"TIMER_DRIVEN\", \"executionNode\": \"PRIMARY\", \"concurrentlySchedulableTaskCount\": \"1\", \"properties\": \"{\\\"Bucket\\\": \\\"enriched-topical-chat\\\", \\\"Region\\\": \\\"us-west-2\\\", \\\"Access Key\\\": null, \\\"Secret Key\\\": null, \\\"record-writer\\\": null, \\\"min-age\\\": \\\"0 sec\\\", \\\"Listing Batch Size\\\": \\\"100\\\", \\\"write-s3-object-tags\\\": \\\"false\\\", \\\"write-s3-user-metadata\\\": \\\"false\\\", \\\"Credentials File\\\": null, \\\"AWS Credentials Provider service\\\": null, \\\"Communications Timeout\\\": \\\"30 secs\\\", \\\"SSL Context Service\\\": null, \\\"Endpoint Override URL\\\": null, \\\"Signer Override\\\": \\\"Default Signature\\\", \\\"proxy-configuration-service\\\": null, \\\"Proxy Host\\\": null, \\\"Proxy Host Port\\\": null, \\\"proxy-user-name\\\": null, \\\"proxy-user-password\\\": null, \\\"delimiter\\\": null, \\\"prefix\\\": null, \\\"use-versions\\\": \\\"false\\\", \\\"list-type\\\": \\\"1\\\", \\\"requester-pays\\\": \\\"false\\\"}\", \"last_event_time\": \"2022-08-23 19:44:39.852000+00:00\"}, \"externalUrl\": \"http://localhost:9080/nifi/?processGroupId=1c61a8d9-3462-387f-8145-09e6e7785e5c&componentIds=8a218b6e-e6a0-36b6-bc4b-79d202a80167\", \"name\": \"ListS3\", \"description\": \"\", \"type\": {\"string\": \"NIFI_PROCESSOR\"}}", - "contentType": "application/json" + "json": { + "customProperties": { + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "PRIMARY", + "concurrentlySchedulableTaskCount": "1", + "properties": "{\"listing-strategy\": \"timestamps\", \"et-state-cache\": null, \"et-initial-listing-target\": \"all\", \"et-time-window\": \"3 hours\", \"Bucket\": \"enriched-topical-chat\", \"Region\": \"us-west-2\", \"Access Key\": null, \"Secret Key\": null, \"record-writer\": null, \"min-age\": \"0 sec\", \"max-age\": null, \"Listing Batch Size\": \"100\", \"write-s3-object-tags\": \"false\", \"write-s3-user-metadata\": \"false\", \"Credentials File\": null, \"AWS Credentials Provider service\": null, \"Communications Timeout\": \"30 secs\", \"SSL Context Service\": null, \"Endpoint Override URL\": null, \"Signer Override\": \"Default Signature\", \"custom-signer-class-name\": null, \"custom-signer-module-location\": null, \"proxy-configuration-service\": null, \"Proxy Host\": null, \"Proxy Host Port\": null, \"proxy-user-name\": null, \"proxy-user-password\": null, \"delimiter\": null, \"prefix\": null, \"use-versions\": \"false\", \"list-type\": \"1\", \"requester-pays\": \"false\"}", + "last_event_time": "2023-07-05 16:10:34.569000+00:00" + }, + "externalUrl": "http://localhost:9080/nifi/?processGroupId=1c61a8d9-3462-387f-8145-09e6e7785e5c&componentIds=8a218b6e-e6a0-36b6-bc4b-79d202a80167", + "name": "ListS3", + "description": "", + "type": { + "string": "NIFI_PROCESSOR" + } + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),8a218b6e-e6a0-36b6-bc4b-79d202a80167)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataJobInputOutput", "aspect": { - "value": "{\"inputDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)\"], \"outputDatasets\": [], \"inputDatajobs\": []}", - "contentType": "application/json" + "json": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)" + ], + "outputDatasets": [], + "inputDatajobs": [] + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),71bc17ed-a3bc-339a-a100-ebad434717d4)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataJobInfo", "aspect": { - "value": "{\"customProperties\": {}, \"externalUrl\": \"http://localhost:9080/nifi/?processGroupId=1c61a8d9-3462-387f-8145-09e6e7785e5c&componentIds=8efa023d-017d-1000-0000-0000479b764f\", \"name\": \"s3_data\", \"description\": \"\", \"type\": {\"string\": \"NIFI_REMOTE_INPUT_PORT\"}}", - "contentType": "application/json" + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9080/nifi/?processGroupId=1c61a8d9-3462-387f-8145-09e6e7785e5c&componentIds=8efa023d-017d-1000-0000-0000479b764f", + "name": "s3_data", + "description": "", + "type": { + "string": "NIFI_REMOTE_INPUT_PORT" + } + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),71bc17ed-a3bc-339a-a100-ebad434717d4)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataJobInputOutput", "aspect": { - "value": "{\"inputDatasets\": [], \"outputDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:nifi,default.s3_data,PROD)\"], \"inputDatajobs\": [\"urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),c5f6fc66-ffbb-3f60-9564-f2466ae32493)\"]}", - "contentType": "application/json" + "json": { + "inputDatasets": [], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:nifi,default.s3_data,PROD)" + ], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),c5f6fc66-ffbb-3f60-9564-f2466ae32493)" + ] + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host.temperature,PROD)", - "entityKeyAspect": null, + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),c8c73d4c-ebdd-1bee-9b46-629672cd11a0)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "dataJobInfo", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", - "contentType": "application/json" + "json": { + "customProperties": { + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "concurrentlySchedulableTaskCount": "1", + "properties": "{\"Object Key\": \"sftp_data/${filename}\", \"Bucket\": \"${s3.destbucket}\", \"Content Type\": null, \"Content Disposition\": null, \"Cache Control\": null, \"Access Key\": null, \"Secret Key\": null, \"Credentials File\": null, \"AWS Credentials Provider service\": \"c8c73d64-ebdd-1bee-0000-000020079e12\", \"s3-object-tags-prefix\": null, \"s3-object-remove-tags-prefix\": \"false\", \"Storage Class\": \"Standard\", \"Region\": \"us-east-1\", \"Communications Timeout\": \"30 secs\", \"Expiration Time Rule\": null, \"FullControl User List\": \"${s3.permissions.full.users}\", \"Read Permission User List\": \"${s3.permissions.read.users}\", \"Write Permission User List\": \"${s3.permissions.write.users}\", \"Read ACL User List\": \"${s3.permissions.readacl.users}\", \"Write ACL User List\": \"${s3.permissions.writeacl.users}\", \"Owner\": \"${s3.owner}\", \"canned-acl\": \"${s3.permissions.cannedacl}\", \"SSL Context Service\": null, \"Endpoint Override URL\": null, \"Signer Override\": \"Default Signature\", \"custom-signer-class-name\": null, \"custom-signer-module-location\": null, \"Multipart Threshold\": \"5 GB\", \"Multipart Part Size\": \"5 GB\", \"Multipart Upload AgeOff Interval\": \"60 min\", \"Multipart Upload Max Age Threshold\": \"7 days\", \"s3-temporary-directory-multipart\": \"${java.io.tmpdir}\", \"server-side-encryption\": \"None\", \"encryption-service\": null, \"use-chunked-encoding\": \"true\", \"use-path-style-access\": \"false\", \"proxy-configuration-service\": null, \"Proxy Host\": null, \"Proxy Host Port\": null, \"proxy-user-name\": null, \"proxy-user-password\": null}", + "last_event_time": "None" + }, + "externalUrl": "http://localhost:9080/nifi/?processGroupId=8eb4f034-017d-1000-ffff-ffffccebd06c&componentIds=c8c73d4c-ebdd-1bee-9b46-629672cd11a0", + "name": "PutS3Object", + "description": "", + "type": { + "string": "NIFI_PROCESSOR" + } + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host.temperature,PROD)", - "entityKeyAspect": null, + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),c8c73d4c-ebdd-1bee-9b46-629672cd11a0)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "dataJobInputOutput", "aspect": { - "value": "{\"customProperties\": {\"uri\": \"sftp://sftp_public_host/temperature\"}, \"tags\": []}", - "contentType": "application/json" + "json": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),3ec2acd6-a0d4-3198-9066-a59fb757bc05)" + ] + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host,PROD)", - "entityKeyAspect": null, + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host.temperature,PROD)", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", - "contentType": "application/json" + "json": { + "platform": "urn:li:dataPlatform:file" + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host,PROD)", - "entityKeyAspect": null, + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host.temperature,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { - "value": "{\"customProperties\": {\"uri\": \"sftp://sftp_public_host\"}, \"tags\": []}", - "contentType": "application/json" + "json": { + "customProperties": { + "uri": "sftp://sftp_public_host/temperature" + }, + "tags": [] + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),8eb5263d-017d-1000-ffff-ffff911b23aa)", - "entityKeyAspect": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host,PROD)", "changeType": "UPSERT", - "aspectName": "dataJobInfo", + "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"customProperties\": {\"schedulingPeriod\": \"0 sec\", \"schedulingStrategy\": \"TIMER_DRIVEN\", \"executionNode\": \"PRIMARY\", \"concurrentlySchedulableTaskCount\": \"1\", \"properties\": \"{\\\"listing-strategy\\\": \\\"timestamps\\\", \\\"Hostname\\\": \\\"${sftp.host}\\\", \\\"Port\\\": \\\"22\\\", \\\"Username\\\": \\\"${sftp.username}\\\", \\\"Password\\\": \\\"********\\\", \\\"Private Key Path\\\": null, \\\"Private Key Passphrase\\\": null, \\\"Remote Path\\\": \\\".\\\", \\\"record-writer\\\": null, \\\"Distributed Cache Service\\\": null, \\\"Search Recursively\\\": \\\"true\\\", \\\"follow-symlink\\\": \\\"false\\\", \\\"File Filter Regex\\\": null, \\\"Path Filter Regex\\\": null, \\\"Ignore Dotted Files\\\": \\\"true\\\", \\\"Strict Host Key Checking\\\": \\\"false\\\", \\\"Host Key File\\\": null, \\\"Connection Timeout\\\": \\\"30 sec\\\", \\\"Data Timeout\\\": \\\"30 sec\\\", \\\"Send Keep Alive On Timeout\\\": \\\"true\\\", \\\"target-system-timestamp-precision\\\": \\\"auto-detect\\\", \\\"proxy-configuration-service\\\": null, \\\"Proxy Type\\\": \\\"DIRECT\\\", \\\"Proxy Host\\\": null, \\\"Proxy Port\\\": null, \\\"Http Proxy Username\\\": null, \\\"Http Proxy Password\\\": null, \\\"et-state-cache\\\": null, \\\"et-time-window\\\": \\\"3 hours\\\", \\\"et-initial-listing-target\\\": \\\"all\\\", \\\"Minimum File Age\\\": \\\"0 sec\\\", \\\"Maximum File Age\\\": null, \\\"Minimum File Size\\\": \\\"0 B\\\", \\\"Maximum File Size\\\": null, \\\"Ciphers Allowed\\\": null, \\\"Key Algorithms Allowed\\\": null, \\\"Key Exchange Algorithms Allowed\\\": null, \\\"Message Authentication Codes Allowed\\\": null}\", \"last_event_time\": \"2022-08-23 19:44:15.866000+00:00\"}, \"externalUrl\": \"http://localhost:9080/nifi/?processGroupId=8eb4f034-017d-1000-ffff-ffffccebd06c&componentIds=8eb5263d-017d-1000-ffff-ffff911b23aa\", \"name\": \"ListSFTP\", \"description\": \"\", \"type\": {\"string\": \"NIFI_PROCESSOR\"}}", - "contentType": "application/json" + "json": { + "platform": "urn:li:dataPlatform:file" + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),8eb5263d-017d-1000-ffff-ffff911b23aa)", - "entityKeyAspect": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host,PROD)", "changeType": "UPSERT", - "aspectName": "dataJobInputOutput", + "aspectName": "datasetProperties", "aspect": { - "value": "{\"inputDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host.temperature,PROD)\"], \"outputDatasets\": [], \"inputDatajobs\": []}", - "contentType": "application/json" + "json": { + "customProperties": { + "uri": "sftp://sftp_public_host" + }, + "tags": [] + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),c8c73d4c-ebdd-1bee-9b46-629672cd11a0)", - "entityKeyAspect": null, + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),8eb5263d-017d-1000-ffff-ffff911b23aa)", "changeType": "UPSERT", "aspectName": "dataJobInfo", "aspect": { - "value": "{\"customProperties\": {\"schedulingPeriod\": \"0 sec\", \"schedulingStrategy\": \"TIMER_DRIVEN\", \"executionNode\": \"ALL\", \"concurrentlySchedulableTaskCount\": \"1\", \"properties\": \"{\\\"Object Key\\\": \\\"sftp_data/${filename}\\\", \\\"Bucket\\\": \\\"${s3.destbucket}\\\", \\\"Content Type\\\": null, \\\"Content Disposition\\\": null, \\\"Cache Control\\\": null, \\\"Access Key\\\": null, \\\"Secret Key\\\": null, \\\"Credentials File\\\": null, \\\"AWS Credentials Provider service\\\": \\\"c8c73d64-ebdd-1bee-0000-000020079e12\\\", \\\"s3-object-tags-prefix\\\": null, \\\"s3-object-remove-tags-prefix\\\": \\\"false\\\", \\\"Storage Class\\\": \\\"Standard\\\", \\\"Region\\\": \\\"us-east-1\\\", \\\"Communications Timeout\\\": \\\"30 secs\\\", \\\"Expiration Time Rule\\\": null, \\\"FullControl User List\\\": \\\"${s3.permissions.full.users}\\\", \\\"Read Permission User List\\\": \\\"${s3.permissions.read.users}\\\", \\\"Write Permission User List\\\": \\\"${s3.permissions.write.users}\\\", \\\"Read ACL User List\\\": \\\"${s3.permissions.readacl.users}\\\", \\\"Write ACL User List\\\": \\\"${s3.permissions.writeacl.users}\\\", \\\"Owner\\\": \\\"${s3.owner}\\\", \\\"canned-acl\\\": \\\"${s3.permissions.cannedacl}\\\", \\\"SSL Context Service\\\": null, \\\"Endpoint Override URL\\\": null, \\\"Signer Override\\\": \\\"Default Signature\\\", \\\"Multipart Threshold\\\": \\\"5 GB\\\", \\\"Multipart Part Size\\\": \\\"5 GB\\\", \\\"Multipart Upload AgeOff Interval\\\": \\\"60 min\\\", \\\"Multipart Upload Max Age Threshold\\\": \\\"7 days\\\", \\\"s3-temporary-directory-multipart\\\": \\\"${java.io.tmpdir}\\\", \\\"server-side-encryption\\\": \\\"None\\\", \\\"encryption-service\\\": null, \\\"use-chunked-encoding\\\": \\\"true\\\", \\\"use-path-style-access\\\": \\\"false\\\", \\\"proxy-configuration-service\\\": null, \\\"Proxy Host\\\": null, \\\"Proxy Host Port\\\": null, \\\"proxy-user-name\\\": null, \\\"proxy-user-password\\\": null}\", \"last_event_time\": \"None\"}, \"externalUrl\": \"http://localhost:9080/nifi/?processGroupId=8eb4f034-017d-1000-ffff-ffffccebd06c&componentIds=c8c73d4c-ebdd-1bee-9b46-629672cd11a0\", \"name\": \"PutS3Object\", \"description\": \"\", \"type\": {\"string\": \"NIFI_PROCESSOR\"}}", - "contentType": "application/json" + "json": { + "customProperties": { + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "PRIMARY", + "concurrentlySchedulableTaskCount": "1", + "properties": "{\"listing-strategy\": \"timestamps\", \"Hostname\": \"${sftp.host}\", \"Port\": \"22\", \"Username\": \"${sftp.username}\", \"Password\": \"********\", \"Private Key Path\": null, \"Private Key Passphrase\": null, \"Remote Path\": \".\", \"record-writer\": null, \"Distributed Cache Service\": null, \"Search Recursively\": \"true\", \"follow-symlink\": \"false\", \"File Filter Regex\": null, \"Path Filter Regex\": null, \"Ignore Dotted Files\": \"true\", \"Strict Host Key Checking\": \"false\", \"Host Key File\": null, \"Connection Timeout\": \"30 sec\", \"Data Timeout\": \"30 sec\", \"Send Keep Alive On Timeout\": \"true\", \"target-system-timestamp-precision\": \"auto-detect\", \"Use Compression\": \"false\", \"proxy-configuration-service\": null, \"Proxy Type\": \"DIRECT\", \"Proxy Host\": null, \"Proxy Port\": null, \"Http Proxy Username\": null, \"Http Proxy Password\": null, \"et-state-cache\": null, \"et-time-window\": \"3 hours\", \"et-initial-listing-target\": \"all\", \"Minimum File Age\": \"0 sec\", \"Maximum File Age\": null, \"Minimum File Size\": \"0 B\", \"Maximum File Size\": null, \"Ciphers Allowed\": null, \"Key Algorithms Allowed\": null, \"Key Exchange Algorithms Allowed\": null, \"Message Authentication Codes Allowed\": null}", + "last_event_time": "2023-07-05 16:10:34.541000+00:00" + }, + "externalUrl": "http://localhost:9080/nifi/?processGroupId=8eb4f034-017d-1000-ffff-ffffccebd06c&componentIds=8eb5263d-017d-1000-ffff-ffff911b23aa", + "name": "ListSFTP", + "description": "", + "type": { + "string": "NIFI_PROCESSOR" + } + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),c8c73d4c-ebdd-1bee-9b46-629672cd11a0)", - "entityKeyAspect": null, + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),8eb5263d-017d-1000-ffff-ffff911b23aa)", "changeType": "UPSERT", "aspectName": "dataJobInputOutput", "aspect": { - "value": "{\"inputDatasets\": [], \"outputDatasets\": [], \"inputDatajobs\": [\"urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),3ec2acd6-a0d4-3198-9066-a59fb757bc05)\"]}", - "contentType": "application/json" + "json": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host.temperature,PROD)" + ], + "outputDatasets": [], + "inputDatajobs": [] + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host.temperature,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", - "contentType": "application/json" + "json": { + "platform": "urn:li:dataPlatform:file" + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host.temperature,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { - "value": "{\"customProperties\": {\"uri\": \"sftp://sftp_public_host/temperature\"}, \"tags\": []}", - "contentType": "application/json" + "json": { + "customProperties": { + "uri": "sftp://sftp_public_host/temperature" + }, + "tags": [] + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", - "contentType": "application/json" + "json": { + "platform": "urn:li:dataPlatform:file" + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { - "value": "{\"customProperties\": {\"uri\": \"sftp://sftp_public_host\"}, \"tags\": []}", - "contentType": "application/json" + "json": { + "customProperties": { + "uri": "sftp://sftp_public_host" + }, + "tags": [] + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),8eb55aeb-017d-1000-ffff-fffff475768d)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataJobInfo", "aspect": { - "value": "{\"customProperties\": {\"schedulingPeriod\": \"0 sec\", \"schedulingStrategy\": \"TIMER_DRIVEN\", \"executionNode\": \"ALL\", \"concurrentlySchedulableTaskCount\": \"1\", \"properties\": \"{\\\"Hostname\\\": \\\"${sftp.host}\\\", \\\"Port\\\": \\\"22\\\", \\\"Username\\\": \\\"${sftp.username}\\\", \\\"Password\\\": \\\"********\\\", \\\"Private Key Path\\\": null, \\\"Private Key Passphrase\\\": null, \\\"Remote File\\\": \\\"${path}/${filename}\\\", \\\"Completion Strategy\\\": \\\"None\\\", \\\"Move Destination Directory\\\": null, \\\"Create Directory\\\": \\\"false\\\", \\\"Disable Directory Listing\\\": \\\"false\\\", \\\"Connection Timeout\\\": \\\"30 sec\\\", \\\"Data Timeout\\\": \\\"30 sec\\\", \\\"Send Keep Alive On Timeout\\\": \\\"true\\\", \\\"Host Key File\\\": null, \\\"Strict Host Key Checking\\\": \\\"false\\\", \\\"Use Compression\\\": \\\"false\\\", \\\"proxy-configuration-service\\\": null, \\\"Proxy Type\\\": \\\"DIRECT\\\", \\\"Proxy Host\\\": null, \\\"Proxy Port\\\": null, \\\"Http Proxy Username\\\": null, \\\"Http Proxy Password\\\": null, \\\"fetchfiletransfer-notfound-loglevel\\\": \\\"ERROR\\\", \\\"Ciphers Allowed\\\": null, \\\"Key Algorithms Allowed\\\": null, \\\"Key Exchange Algorithms Allowed\\\": null, \\\"Message Authentication Codes Allowed\\\": null}\", \"last_event_time\": \"2022-08-23 19:44:20.256000+00:00\"}, \"externalUrl\": \"http://localhost:9080/nifi/?processGroupId=8eb4f034-017d-1000-ffff-ffffccebd06c&componentIds=8eb55aeb-017d-1000-ffff-fffff475768d\", \"name\": \"FetchSFTP\", \"description\": \"\", \"type\": {\"string\": \"NIFI_PROCESSOR\"}}", - "contentType": "application/json" + "json": { + "customProperties": { + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "concurrentlySchedulableTaskCount": "1", + "properties": "{\"Hostname\": \"${sftp.host}\", \"Port\": \"22\", \"Username\": \"${sftp.username}\", \"Password\": \"********\", \"Private Key Path\": null, \"Private Key Passphrase\": null, \"Remote File\": \"${path}/${filename}\", \"Completion Strategy\": \"None\", \"Move Destination Directory\": null, \"Create Directory\": \"false\", \"Disable Directory Listing\": \"false\", \"Connection Timeout\": \"30 sec\", \"Data Timeout\": \"30 sec\", \"Send Keep Alive On Timeout\": \"true\", \"Host Key File\": null, \"Strict Host Key Checking\": \"false\", \"Use Compression\": \"false\", \"proxy-configuration-service\": null, \"Proxy Type\": \"DIRECT\", \"Proxy Host\": null, \"Proxy Port\": null, \"Http Proxy Username\": null, \"Http Proxy Password\": null, \"fetchfiletransfer-notfound-loglevel\": \"ERROR\", \"Ciphers Allowed\": null, \"Key Algorithms Allowed\": null, \"Key Exchange Algorithms Allowed\": null, \"Message Authentication Codes Allowed\": null}", + "last_event_time": "2023-07-05 16:10:35.274000+00:00" + }, + "externalUrl": "http://localhost:9080/nifi/?processGroupId=8eb4f034-017d-1000-ffff-ffffccebd06c&componentIds=8eb55aeb-017d-1000-ffff-fffff475768d", + "name": "FetchSFTP", + "description": "", + "type": { + "string": "NIFI_PROCESSOR" + } + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),8eb55aeb-017d-1000-ffff-fffff475768d)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataJobInputOutput", "aspect": { - "value": "{\"inputDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host,PROD)\", \"urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host.temperature,PROD)\"], \"outputDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:nifi,default.sftp_files_out,PROD)\"], \"inputDatajobs\": [\"urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),8eb5263d-017d-1000-ffff-ffff911b23aa)\"]}", - "contentType": "application/json" + "json": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host.temperature,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:nifi,default.sftp_files_out,PROD)" + ], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),8eb5263d-017d-1000-ffff-ffff911b23aa)" + ] + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),3ec2acd6-a0d4-3198-9066-a59fb757bc05)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataJobInfo", "aspect": { - "value": "{\"customProperties\": {}, \"externalUrl\": \"http://localhost:9080/nifi/?processGroupId=8eb4f034-017d-1000-ffff-ffffccebd06c&componentIds=8eb70d94-017d-1000-ffff-ffffc94c12ce\", \"name\": \"sftp_files_out\", \"description\": \"\", \"type\": {\"string\": \"NIFI_REMOTE_OUTPUT_PORT\"}}", - "contentType": "application/json" + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9080/nifi/?processGroupId=8eb4f034-017d-1000-ffff-ffffccebd06c&componentIds=8eb70d94-017d-1000-ffff-ffffc94c12ce", + "name": "sftp_files_out", + "description": "", + "type": { + "string": "NIFI_REMOTE_OUTPUT_PORT" + } + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),3ec2acd6-a0d4-3198-9066-a59fb757bc05)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataJobInputOutput", "aspect": { - "value": "{\"inputDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:nifi,default.sftp_files_out,PROD)\"], \"outputDatasets\": [], \"inputDatajobs\": []}", - "contentType": "application/json" + "json": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:nifi,default.sftp_files_out,PROD)" + ], + "outputDatasets": [], + "inputDatajobs": [] + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:nifi,default.s3_data,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:nifi\"}", - "contentType": "application/json" + "json": { + "platform": "urn:li:dataPlatform:nifi" + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:nifi,default.s3_data,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { - "value": "{\"customProperties\": {}, \"externalUrl\": \"http://localhost:9080/nifi/?processGroupId=1c61a8d9-3462-387f-8145-09e6e7785e5c&componentIds=8ef96dcf-017d-1000-ffff-ffff8f7528f0\", \"tags\": []}", - "contentType": "application/json" + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9080/nifi/?processGroupId=1c61a8d9-3462-387f-8145-09e6e7785e5c&componentIds=8ef96dcf-017d-1000-ffff-ffff8f7528f0", + "tags": [] + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:nifi,default.sftp_files_out,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:nifi\"}", - "contentType": "application/json" + "json": { + "platform": "urn:li:dataPlatform:nifi" + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:nifi,default.sftp_files_out,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { - "value": "{\"customProperties\": {}, \"externalUrl\": \"http://localhost:9080/nifi/?processGroupId=8eb4f034-017d-1000-ffff-ffffccebd06c&componentIds=8eb66675-017d-1000-ffff-ffffa56e2758\", \"tags\": []}", - "contentType": "application/json" + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9080/nifi/?processGroupId=8eb4f034-017d-1000-ffff-ffffccebd06c&componentIds=8eb66675-017d-1000-ffff-ffffa56e2758", + "tags": [] + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-cluster", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-cluster" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_standalone.json b/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_standalone.json index be4c52d122773..5fe65a651ee65 100644 --- a/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_standalone.json +++ b/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_standalone.json @@ -1,211 +1,231 @@ [ { - "auditHeader": null, "entityType": "dataFlow", "entityUrn": "urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataFlowInfo", "aspect": { - "value": "{\"customProperties\": {\"clustered\": \"False\", \"version\": \"1.15.0\"}, \"externalUrl\": \"http://localhost:9443/nifi/?processGroupId=803ebb92-017d-1000-2961-4bdaa27a3ba0&componentIds=\", \"name\": \"Standalone Flow\"}", - "contentType": "application/json" + "json": { + "customProperties": { + "clustered": "False", + "version": "1.22.0" + }, + "externalUrl": "http://localhost:9443/nifi/?processGroupId=803ebb92-017d-1000-2961-4bdaa27a3ba0&componentIds=", + "name": "Standalone Flow" + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-standalone", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-standalone" } }, { - "auditHeader": null, "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),aed63edf-e660-3f29-b56b-192cf6286889)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataJobInfo", "aspect": { - "value": "{\"customProperties\": {\"schedulingPeriod\": \"0 sec\", \"schedulingStrategy\": \"TIMER_DRIVEN\", \"executionNode\": \"ALL\", \"concurrentlySchedulableTaskCount\": \"1\", \"properties\": \"{\\\"Object Key\\\": \\\"tropical_data/${filename}\\\", \\\"Bucket\\\": \\\"${s3.destbucket}\\\", \\\"Content Type\\\": null, \\\"Content Disposition\\\": null, \\\"Cache Control\\\": null, \\\"Access Key\\\": null, \\\"Secret Key\\\": null, \\\"Credentials File\\\": null, \\\"AWS Credentials Provider service\\\": \\\"80436b00-017d-1000-54c8-ff854b5c8990\\\", \\\"s3-object-tags-prefix\\\": null, \\\"s3-object-remove-tags-prefix\\\": \\\"false\\\", \\\"Storage Class\\\": \\\"Standard\\\", \\\"Region\\\": \\\"us-east-1\\\", \\\"Communications Timeout\\\": \\\"30 secs\\\", \\\"Expiration Time Rule\\\": null, \\\"FullControl User List\\\": \\\"${s3.permissions.full.users}\\\", \\\"Read Permission User List\\\": \\\"${s3.permissions.read.users}\\\", \\\"Write Permission User List\\\": \\\"${s3.permissions.write.users}\\\", \\\"Read ACL User List\\\": \\\"${s3.permissions.readacl.users}\\\", \\\"Write ACL User List\\\": \\\"${s3.permissions.writeacl.users}\\\", \\\"Owner\\\": \\\"${s3.owner}\\\", \\\"canned-acl\\\": \\\"${s3.permissions.cannedacl}\\\", \\\"SSL Context Service\\\": null, \\\"Endpoint Override URL\\\": null, \\\"Signer Override\\\": \\\"Default Signature\\\", \\\"Multipart Threshold\\\": \\\"5 GB\\\", \\\"Multipart Part Size\\\": \\\"5 GB\\\", \\\"Multipart Upload AgeOff Interval\\\": \\\"60 min\\\", \\\"Multipart Upload Max Age Threshold\\\": \\\"7 days\\\", \\\"s3-temporary-directory-multipart\\\": \\\"${java.io.tmpdir}\\\", \\\"server-side-encryption\\\": \\\"None\\\", \\\"encryption-service\\\": null, \\\"use-chunked-encoding\\\": \\\"true\\\", \\\"use-path-style-access\\\": \\\"false\\\", \\\"proxy-configuration-service\\\": null, \\\"Proxy Host\\\": null, \\\"Proxy Host Port\\\": null, \\\"proxy-user-name\\\": null, \\\"proxy-user-password\\\": null}\", \"last_event_time\": \"None\"}, \"externalUrl\": \"http://localhost:9443/nifi/?processGroupId=80404c81-017d-1000-e8e8-af7420af06c1&componentIds=aed63edf-e660-3f29-b56b-192cf6286889\", \"name\": \"PutS3Object\", \"description\": \"\", \"type\": {\"string\": \"NIFI_PROCESSOR\"}}", - "contentType": "application/json" + "json": { + "customProperties": { + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "concurrentlySchedulableTaskCount": "1", + "properties": "{\"Object Key\": \"tropical_data/${filename}\", \"Bucket\": \"${s3.destbucket}\", \"Content Type\": null, \"Content Disposition\": null, \"Cache Control\": null, \"Access Key\": null, \"Secret Key\": null, \"Credentials File\": null, \"AWS Credentials Provider service\": \"80436b00-017d-1000-54c8-ff854b5c8990\", \"s3-object-tags-prefix\": null, \"s3-object-remove-tags-prefix\": \"false\", \"Storage Class\": \"Standard\", \"Region\": \"us-east-1\", \"Communications Timeout\": \"30 secs\", \"Expiration Time Rule\": null, \"FullControl User List\": \"${s3.permissions.full.users}\", \"Read Permission User List\": \"${s3.permissions.read.users}\", \"Write Permission User List\": \"${s3.permissions.write.users}\", \"Read ACL User List\": \"${s3.permissions.readacl.users}\", \"Write ACL User List\": \"${s3.permissions.writeacl.users}\", \"Owner\": \"${s3.owner}\", \"canned-acl\": \"${s3.permissions.cannedacl}\", \"SSL Context Service\": null, \"Endpoint Override URL\": null, \"Signer Override\": \"Default Signature\", \"custom-signer-class-name\": null, \"custom-signer-module-location\": null, \"Multipart Threshold\": \"5 GB\", \"Multipart Part Size\": \"5 GB\", \"Multipart Upload AgeOff Interval\": \"60 min\", \"Multipart Upload Max Age Threshold\": \"7 days\", \"s3-temporary-directory-multipart\": \"${java.io.tmpdir}\", \"server-side-encryption\": \"None\", \"encryption-service\": null, \"use-chunked-encoding\": \"true\", \"use-path-style-access\": \"false\", \"proxy-configuration-service\": null, \"Proxy Host\": null, \"Proxy Host Port\": null, \"proxy-user-name\": null, \"proxy-user-password\": null}", + "last_event_time": "None" + }, + "externalUrl": "http://localhost:9443/nifi/?processGroupId=80404c81-017d-1000-e8e8-af7420af06c1&componentIds=aed63edf-e660-3f29-b56b-192cf6286889", + "name": "PutS3Object", + "description": "", + "type": { + "string": "NIFI_PROCESSOR" + } + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-standalone", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-standalone" } }, { - "auditHeader": null, "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),aed63edf-e660-3f29-b56b-192cf6286889)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataJobInputOutput", "aspect": { - "value": "{\"inputDatasets\": [], \"outputDatasets\": [], \"inputDatajobs\": [\"urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),91d59f03-1c2b-3f3f-48bc-f89296a328bd)\"]}", - "contentType": "application/json" + "json": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),91d59f03-1c2b-3f3f-48bc-f89296a328bd)" + ] + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-standalone", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-standalone" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", - "contentType": "application/json" + "json": { + "platform": "urn:li:dataPlatform:s3" + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-standalone", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-standalone" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { - "value": "{\"customProperties\": {\"s3_uri\": \"s3://enriched-topical-chat\"}, \"tags\": []}", - "contentType": "application/json" + "json": { + "customProperties": { + "s3_uri": "s3://enriched-topical-chat" + }, + "tags": [] + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-standalone", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-standalone" } }, { - "auditHeader": null, "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),91d59f03-1c2b-3f3f-48bc-f89296a328bd)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataJobInfo", "aspect": { - "value": "{\"customProperties\": {\"schedulingPeriod\": \"0 sec\", \"schedulingStrategy\": \"TIMER_DRIVEN\", \"executionNode\": \"ALL\", \"concurrentlySchedulableTaskCount\": \"1\", \"properties\": \"{\\\"Bucket\\\": \\\"enriched-topical-chat\\\", \\\"Object Key\\\": \\\"${filename}\\\", \\\"Region\\\": \\\"us-west-2\\\", \\\"Access Key\\\": null, \\\"Secret Key\\\": null, \\\"Credentials File\\\": null, \\\"AWS Credentials Provider service\\\": null, \\\"Communications Timeout\\\": \\\"30 secs\\\", \\\"Version\\\": null, \\\"SSL Context Service\\\": null, \\\"Endpoint Override URL\\\": null, \\\"Signer Override\\\": \\\"Default Signature\\\", \\\"encryption-service\\\": null, \\\"proxy-configuration-service\\\": null, \\\"Proxy Host\\\": null, \\\"Proxy Host Port\\\": null, \\\"proxy-user-name\\\": null, \\\"proxy-user-password\\\": null, \\\"requester-pays\\\": \\\"false\\\", \\\"range-start\\\": null, \\\"range-length\\\": null}\", \"last_event_time\": \"2022-08-23 19:43:21.880000+00:00\"}, \"externalUrl\": \"http://localhost:9443/nifi/?processGroupId=80404c81-017d-1000-e8e8-af7420af06c1&componentIds=91d59f03-1c2b-3f3f-48bc-f89296a328bd\", \"name\": \"FetchS3Object\", \"description\": \"\", \"type\": {\"string\": \"NIFI_PROCESSOR\"}}", - "contentType": "application/json" + "json": { + "customProperties": { + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "concurrentlySchedulableTaskCount": "1", + "properties": "{\"Bucket\": \"enriched-topical-chat\", \"Object Key\": \"${filename}\", \"Region\": \"us-west-2\", \"Access Key\": null, \"Secret Key\": null, \"Credentials File\": null, \"AWS Credentials Provider service\": null, \"Communications Timeout\": \"30 secs\", \"Version\": null, \"SSL Context Service\": null, \"Endpoint Override URL\": null, \"Signer Override\": \"Default Signature\", \"custom-signer-class-name\": null, \"custom-signer-module-location\": null, \"encryption-service\": null, \"proxy-configuration-service\": null, \"Proxy Host\": null, \"Proxy Host Port\": null, \"proxy-user-name\": null, \"proxy-user-password\": null, \"requester-pays\": \"false\", \"range-start\": null, \"range-length\": null}", + "last_event_time": "2023-07-05 16:09:45.347000+00:00" + }, + "externalUrl": "http://localhost:9443/nifi/?processGroupId=80404c81-017d-1000-e8e8-af7420af06c1&componentIds=91d59f03-1c2b-3f3f-48bc-f89296a328bd", + "name": "FetchS3Object", + "description": "", + "type": { + "string": "NIFI_PROCESSOR" + } + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-standalone", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-standalone" } }, { - "auditHeader": null, "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),91d59f03-1c2b-3f3f-48bc-f89296a328bd)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataJobInputOutput", "aspect": { - "value": "{\"inputDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)\"], \"outputDatasets\": [], \"inputDatajobs\": [\"urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),cb7693ed-f93b-3340-3776-fe80e6283ddc)\"]}", - "contentType": "application/json" + "json": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)" + ], + "outputDatasets": [], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),cb7693ed-f93b-3340-3776-fe80e6283ddc)" + ] + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-standalone", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-standalone" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", - "contentType": "application/json" + "json": { + "platform": "urn:li:dataPlatform:s3" + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-standalone", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-standalone" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { - "value": "{\"customProperties\": {\"s3_uri\": \"s3://enriched-topical-chat\"}, \"tags\": []}", - "contentType": "application/json" + "json": { + "customProperties": { + "s3_uri": "s3://enriched-topical-chat" + }, + "tags": [] + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-standalone", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-standalone" } }, { - "auditHeader": null, "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),cb7693ed-f93b-3340-3776-fe80e6283ddc)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataJobInfo", "aspect": { - "value": "{\"customProperties\": {\"schedulingPeriod\": \"0 sec\", \"schedulingStrategy\": \"TIMER_DRIVEN\", \"executionNode\": \"PRIMARY\", \"concurrentlySchedulableTaskCount\": \"1\", \"properties\": \"{\\\"Bucket\\\": \\\"enriched-topical-chat\\\", \\\"Region\\\": \\\"us-west-2\\\", \\\"Access Key\\\": null, \\\"Secret Key\\\": null, \\\"record-writer\\\": null, \\\"min-age\\\": \\\"0 sec\\\", \\\"Listing Batch Size\\\": \\\"100\\\", \\\"write-s3-object-tags\\\": \\\"false\\\", \\\"write-s3-user-metadata\\\": \\\"false\\\", \\\"Credentials File\\\": null, \\\"AWS Credentials Provider service\\\": null, \\\"Communications Timeout\\\": \\\"30 secs\\\", \\\"SSL Context Service\\\": null, \\\"Endpoint Override URL\\\": null, \\\"Signer Override\\\": \\\"Default Signature\\\", \\\"proxy-configuration-service\\\": null, \\\"Proxy Host\\\": null, \\\"Proxy Host Port\\\": null, \\\"proxy-user-name\\\": null, \\\"proxy-user-password\\\": null, \\\"delimiter\\\": null, \\\"prefix\\\": null, \\\"use-versions\\\": \\\"false\\\", \\\"list-type\\\": \\\"1\\\", \\\"requester-pays\\\": \\\"false\\\"}\", \"last_event_time\": \"2022-08-23 19:43:17.474000+00:00\"}, \"externalUrl\": \"http://localhost:9443/nifi/?processGroupId=80404c81-017d-1000-e8e8-af7420af06c1&componentIds=cb7693ed-f93b-3340-3776-fe80e6283ddc\", \"name\": \"ListS3\", \"description\": \"\", \"type\": {\"string\": \"NIFI_PROCESSOR\"}}", - "contentType": "application/json" + "json": { + "customProperties": { + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "PRIMARY", + "concurrentlySchedulableTaskCount": "1", + "properties": "{\"listing-strategy\": \"timestamps\", \"et-state-cache\": null, \"et-initial-listing-target\": \"all\", \"et-time-window\": \"3 hours\", \"Bucket\": \"enriched-topical-chat\", \"Region\": \"us-west-2\", \"Access Key\": null, \"Secret Key\": null, \"record-writer\": null, \"min-age\": \"0 sec\", \"max-age\": null, \"Listing Batch Size\": \"100\", \"write-s3-object-tags\": \"false\", \"write-s3-user-metadata\": \"false\", \"Credentials File\": null, \"AWS Credentials Provider service\": null, \"Communications Timeout\": \"30 secs\", \"SSL Context Service\": null, \"Endpoint Override URL\": null, \"Signer Override\": \"Default Signature\", \"custom-signer-class-name\": null, \"custom-signer-module-location\": null, \"proxy-configuration-service\": null, \"Proxy Host\": null, \"Proxy Host Port\": null, \"proxy-user-name\": null, \"proxy-user-password\": null, \"delimiter\": null, \"prefix\": null, \"use-versions\": \"false\", \"list-type\": \"1\", \"requester-pays\": \"false\"}", + "last_event_time": "2023-07-05 16:09:42.906000+00:00" + }, + "externalUrl": "http://localhost:9443/nifi/?processGroupId=80404c81-017d-1000-e8e8-af7420af06c1&componentIds=cb7693ed-f93b-3340-3776-fe80e6283ddc", + "name": "ListS3", + "description": "", + "type": { + "string": "NIFI_PROCESSOR" + } + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-standalone", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-standalone" } }, { - "auditHeader": null, "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),cb7693ed-f93b-3340-3776-fe80e6283ddc)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataJobInputOutput", "aspect": { - "value": "{\"inputDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)\"], \"outputDatasets\": [], \"inputDatajobs\": []}", - "contentType": "application/json" + "json": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)" + ], + "outputDatasets": [], + "inputDatajobs": [] + } }, "systemMetadata": { "lastObserved": 1638532800000, - "runId": "nifi-test-standalone", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "nifi-test-standalone" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/nifi/setup/conf_clustered/flow.xml.gz b/metadata-ingestion/tests/integration/nifi/setup/conf_clustered/flow.xml.gz index 5b684d598f04422f3ebb9ccf1344bc943df58120..9a05bb9099a88b3b25b1c45d00106181cb81b4f5 100644 GIT binary patch literal 4673 zcmV-H629#piwFP!000000PQ_%bE7!6-{)6Qsj1rCs+G-N{Vtf4@h7G1y>}fIG=H^C8pdaU>m`x1{6qe>|-m<}2|7+jBpiaXt2o zA+F(>w!8Rrc0a#D>KPM+*fp`^x#ZKC>z(~$@YA{Fc;7BOH}pNnA%4~c@N#D3@MJ4U(lZ73GzD^m zCOupLJGgM70AMp$fR^)qT$I$Vy}%9uA#Mr?u^4x`qME1jIP+S2rjC_>7G&6Q2Y@heb?D<3t^4f}$fqF-@du28(n-v@9JH&9KBgd29$|5`{A%SwoR5 zb_j#xRW+pykuQ)2Gz`LLCzWXxYJ}a=9O7WUX5@g9nh@Xi%ni-%mFG5vxY)7(h$%R&6w+QL zRoi~gHgU|fluO#@i}EN2?HStc4f#r(!NYKRN7tG!+qQMQAeeigY%yp!3p)Wh?-$DU zmP*W7h(j`%-rwC_-~E~`byHr3n^_19OBT=%}MA#Z;gfAXAXzJ%ARH25@U&7J6H`S zqr2hF_5Y0?uWrTWB@kP+BUHBXoBGB|NJKvxU}hifb`=AIl|u`=)l;YBjXF%6D`@eKmQ3F zZ;~L?<93aQiw!9^WaiInvM0}82kO!eHiEDzL@J(`)Ix0tYiLlhS+=Jy1T-TCkZ=ot zE_`AFzHE>J3{|vE7x*w^@*pR^_tiGRoDGQo)i(CPXCx(SmZ6%ejChOXkYKS2(m2*a zyv~WlG*nJew5DVh2*`_>3WazPptVmd`>ElAw=jLO^1hNoptHrMGPl`kW})Zf1yCkc zwv|K8Vu#B&n~pXGCupCQvSFeC5gdewYc2{_tH=eRMmaZNs7L2T)siStqCwM9x8`L$ zA~TH|bHWuU-&ANY_dxy34LFg9B`QE#*#Wg!f}RE4dlB>t8cWwHc(cqTgZ-M>P=~ms z**y5*)HIyVHwvRo>A5PB;V#N(sbt3$M+;d+W-R~~gFkmDsXzai{fK!Cjl?~uuy@Jf71u^-1xb)`!2vPjm`3u&ppAer~L(!QKkV zY7M53|BOuA2g}QYf2}t2Iv_>sS?e)Ud8ag(X9n_))@(q9^5pfOwd0j@n+4y&! z+n67>!=MU#`h@^NSs-H>xnDp8ZvyWJyv?Zb3h01q94=8Dd;+mg{h`y^$x!q3j114U z>_r+JXbF203Yq!M+t<^t2EW8&nZ7 zr^HDt`lY=d1W~%}2invEWMSKh)!cDF@f?dG*s7qcAz89z$TG4lT?bcGW|1bzBC>c< zmN5}{Q4`97vR5QDmbJpNye6knS&3Iq8I|3cQWux);<8;_wqacMNNkk2zD-ey!X!2e z`W1(b-Ld1b;1;95$74O;NDv~v={Q|9v}vsSEn;QAC&IhwmUW)@f0^gqx^vPQ?r&na zS3~s4zav5~rU1a_6MJo7?<@#-i6t)ms~~}DT!hP}p37+*4kBR10#5*$g2p33vWFCX3)<2=)8{~ z!TX@R1z2J>XsVRLF%vSO$NXi5Lu2_D#r{g9eYD>(p^g zeu)Ko@L-3_zn+DlOV0jQ(_1+pa3eQf6ERyB%PQtMRhJ1OGG>t=>oU>}QAdhq@+^;4 zmXnn-u}lx{cCTwz3S;FovN(uMBvR#`-BhY3kAdwdb;#r_GT37)^2l{5aX7N!&VI+ZB zf;MT4ueXTL{kRb`pBUg+qWJpB-i47D!UDcTcu^f?N#%1h6aw3q4e~-LHwp7FpQ+_> z_|%`HPY{~TImRw3USB+|`<;y##}8H{#Po{sX~*n>ug6F2!lnlY{(xcAF3S1NqMRmi z>{Scetd;BK^uA2$G@V%DQ*#i7B@ZyRimY)=x83-0nJaI*>nNl|n(Ihid17g@Y2+W` zTJT@|N3|4*S1q>8l@El+GQC%PV<$*_>df=v?ro0LcQ5$!FLvQx;Sqz??tPn*l^2r4 zW=wBy?U!fB+(X^DAGe#7JZ;UeidkMTb690Je?&#o6|pb~ZBYmq zu!kZB;8MB)NFHifZkV7@w$pSCdj*>ldWPo=ew)uHY4bvPfiIt)sSxk)7SikH`N9$= zN;X{Kbc0jqb!|a6kRWInVc2#=bZm-(r~n!4JXbx!NIUyo@~pkM+d;8(hht{*@nk}u zF;{UC#a!&|Ue^@~S@S%Mm-8bfxPXAkxTY_KX#_0Y-<{H&wA*LLelI)QyHx==`UbhK zd)F>&z6&8n!gbwe_QgKCu;}B%!DKqV7|mwm>85pDx@}4yJ<{MZAo{v4KNS0v!rIKK zowx?B@DWIe#N2VS&KVt{G2FaqEm*dB~@%-^-JiL7TJiHm+U5seB zb-!UwOw#B!1r1_2JJ#%%kCW%;bf}B*?PLmMRT!p(@6Y@Dd|b`itYBCICnF9E77!mR zA=S_YWMM_XtRkAS!o4MH*E`jIudH29Fuj1a>#gHOtesm?O_t6M$~u;Qmn^NwEN`kj zVtL>efdiJ-IaUUxB6Fsx%cg1RZ^_d2PDI};OV<-fAC09uE&aY(8r%$3)y*;kTvsqp zM3KBHOV@IPz8_2H?bI4PT`z&s2jl5}$G&gCzG(u&^i-wD#r3k2>_@tW53}46)aC^0 zVt25;AXrIo>m~O|YF!U9wiBaO@yvSZ$wRO9#6Qx_BI07*A!Zs1-Fy{-VU=eB@j(^J z;q0{0`1g0vf9!NNTYr!8m(_>uqu7dRi-NRC2?ma%RQ`IQD7&mVy_0dL|dB)*FsWH&v#%R>zU zhaRiaqw*i;y1w4cS^1svlq}_2zdbyzG<)jDkUU$DBEvYnL z01>`T^&CTid^#)S?K@*)Ya8SfWuJRDuLSV1iFAV7=hCMzFJ zaeFnN98Geo@NDUw=`P9bJ5O?JsD>h#qJe-GfIxIYkfw_=l6g&5c*Ep4TzdFkH_5Fk zZ?YfrD%<6_?XJ`_%*VM>|6{*qs*%?qu)=kh-*yjFm*4h7Tq@2;Dp7*6m{Wq;$v)4r ziblA`)_0fR_LMI;y8N~UkX?S;^4WCxZ4YCn%Wr#htzCZGA3B_K5O$Z}wqo!uzipS_ zb`M~e-?j<%Cfx(FYKR{AZk(WnPmj`po1&FgEvtW2eb%n`N2W zPN&N*)x~4G6ek}z#R=66WuiRqB3a?OM~~YZ+cjkxC>w}x9E793ZJo75X}jj@snA;4 zfMEFiDG4fOO;wUuNnj0KR}CHOoP;@17fef#S%uemB5|6*@}_ABx}~WyA(CvWq9loe zMifEDSXPLFbxTzQtg42oXqY~M5_GfqDt!AZkQIB04rB=2TR zrL@mQP|w5wmkZ383oTP z5aC}=YfaNL0(wwD-zQJWf;5evQ(|co-u1%ue8OCMA>C9u+iLfs4cvpmw2d&a-6=C& zEQ#^ukQdd;E@Z>CaAb3l-S({+ZErexdO_kl4K*XK33wuFX6OJPGUMZGY=bD0SJA+b zK}c38vjga{kYSf42hT>QjtnwM7+;}itZ13s_3n@DWS5-`XDhh8o{lc&%{6SR-h2y= z{BL2nW)f&sR@3VtR+&X_oPF;*Cc=A@Tz|JPML;hM45M&K z?`I_EEubSz@`52=7)X;a(aQp9mZ(UIteVw76jeWx#!K>PLut7Qb+NQAme$>EY$J}= z1+`8c)Z#QvVX-LE3~YjCS~{;9nm|}f6gW(|YvZdAB|t~>fu25lTI z-+JI4Uv8gYGnbpQ_2w5$>7G=AYu69Gw1fCHPc|HrW+FcAuw4wLFmM|&lr}?LZxf{J z0x(?w<^aT&0x%Z!O{m(5b6Z;U(uWsX^tBywyoEzeJIEG6V{xA~&%v%d#SsMG>$h6JC+j(%i)K zVH9PI;ITwk6+#Ky48>ynR`P>pDBj25xfOw7;_M;x=!4|=k|j!d?;!VRLfZ3wanu@l zl68eOweksRqQUV-Sq`ZBd6HFWlP6i+eE{2~NiL*Ao(><{%932h&&hrZ#Ne@E$_TTj zhy+ekk;cmKDQOlsnu>0)Qbn5NgFIb_IP;s{YPRG$Lh9|ZC6|bX*Q874EuCmAE5I^U zEm@`@>mrgRkQ^efsunN5Wwy3@lyx*)TRnkPBU@XVTzGsxe^J)9dKm#;lsCCf4vH>o z+kaoywo*1u=u2WtAKMbq}P`2cHxo)~_$?rR9T$e4m*{y&sTXLfS zCv*lA+VjHg=N?WHrwYt(%3xq)#%DB18JM0hE=Zw`r9;aTt9NVj41W55?R5_s5wrjR Dtw0)8 literal 61643 zcmagFV~}jkqITQ1x!UGx+qP|UwQbwBZQHhOd$n!de)m55&b?pU`jHW{X3ol}sECRj z&v+6q0s`RQuZu1VhmF>H@;9$fC>8T^`{^?1wCjesM0rZ%@{UozLqApG=X2ml0tuN%UJIf3zV4xqhQ= z&i9b~Oqsp=PuFlXJoJO>KhEEcd~}_mDGd)&in6jVvJG`PH!~E5R5;^qV!<@C2W;Z# zhKZT8Jtww+CFH`;0?h6*4LRs`nP1JJDGqn&Umn=mZRHge-ME0t4g$Gl`@u`` zUjPDlZP3WYlc60izp+3mUPZz0_5tr)4O6^1n|ewbe%Pd8 zu&na0Ohpn{I!qyIa?Sbuc0Y>1WfO$58QqU&_8E-$NI~@wB5&lHdhj!41F1$Y z?PZ-LzazNwAEjy#oyKRKc$ik}5=^3I`6alZ&3XW%SW^oICN-ZTPP)K$A>f_fOUxdD z9z(R_|gWGfrDYXF4iFOGH~KkL&3gA9E1%6+EVgN_sTdX6F8+S2_13@ zn}X~qwv{`?2Ir8y_15(uKMNx>>?031jf=TPK3wqFnnIr*xK^_c)%~I97Iu4@7Hz6K z=fcGPe(>=W-f8jL!hAt26aW4TuNfi77BQ3f??d6;#&Sje4NVXQM~7Ey`yB?ca(Z0P~tC3kNosDUaA1zxn(q(i0=+OlL%#Djf1gbX z<3!eomv^>7b3=c$osmAYH~%6_TByz0CO_OVOgh3P!0ciXzGr6SAioOs=#``?B;Y|u zML!fCQoU}zxtF#B=o&kkS*EY(GdXnHjIm~G8;6u(=$BCT1Z?xova3@428fPn=ZL7N z%PJ+Q1}Ol4aj3?td_4~B=~_7=PFDHQBNW=#_&-i+B-*Ch6?TpyN6u?*WS+wzTBJ;g z;FF)@s+N^Yl1hk{$mY%wEhfNjGPUqEwM5yZYgm-PIEg zVRca_tA+AV6LFkv!cjBb#~lRU2nkNI_uP|Y^l`v3e>uVvX;X>NelhE6$|s$=F26sd z+@!_g*RDq+^`}ga;Hu~-Fl8BX8{7i*-vU+8(|^^N$*4{xf3SYQi5|En%AC0BokWQ( z3N`{jn%MZ7^z9o@&u>P&*6gdB9_z>gvyGcc2CCip@0zi{BTqDJYHAs_Q*-tJrrE+R zknwSoss$q*2vpC9<#6@*;%5sVj7;WVnmt-Wm>fO3zRzx8#MvX;+~3I1O2H}feOn*g}sK1)Wso#GxY52CNyp>!) zLIli#sG`gN$CZOlY74O1u#8VxdgJb4l+`n$h-AlLF09)0#H{_-`wK59scQu6HiccI{Mnv%3I0mF zgwTqTBPN~ws94VkQ)EQ1H&sfu1m5A(g>TmsgS5{pAE@$GW`8U8g6qKk?4cZJr>RFP zkXTq{z142mkKPBrgP`_uLDeP85R$J^)G-qPOx3^a_C`9#*rVbYoj?Jh=lp|Po8E0) zJ)bi`WwYZDq=y%91Knx)!wwDD z6ah#gW*~WR`NknOv~!@cKKVZ1GSy7RTv>FB1I1Yu$X3xqrn&i+_s&fK*MO9C%UW2%&{*Yv-N8N(t!nVDC83wBJtWZ3 zcxt2Mwc?w;E$QDOjS6tJe|rD?lm^pz?W(8F(G7*+UHga|Bv1A+y2x94c4p$8#YQ?X zeF3C{fd>lh;o#$eksffQL$^FcV!kMdnwc79uk^kse6!ey5>kPq!Igg#Rw6{%HO9Vd zA=+yJ(SNj%nQslH&lzit(v|GD_F&DvX(7_dS%a}yQ^sC{Yl4wA?jnKmT)GoT<}99s za{sN7PeQnClnveX$mt=LJ(haJ!GKpK zDzeEWGn{~wF--RXKxT`U+b@5A`?j(4_}Nmka)Q_6#rEy)?Q|RF8r=*iX%NqL24PMW z9FY-={KR$inJMjL=jG-Qu~yCiYq*n>ZFz0@wX70_&%lKBXEv1Xz6{M4cn1K{5vG*G z+>R+lueM3(73lZ3I8jNjlPX`_S*6m$X+de{1 zQjY$-c@juMDraW@rrg(_>yw^q<@fBOmSJL%nR-r*}{Zy?8-)e{4*?B-I;Ej*i zk^T&@M+eOaa7PF2mD^LX&HW=U?h_+zqb-gbdLja@>Br|_ft!;zGsNv(H1c{|fdVpn zt@*ga>6v+O!@EMBUgG-+%4l@Qb-^0Hh!a2=_Z_#@+whT6Nf;*Fy%MKI4|Qr9zIx*y zhJ@TrXAtQ}g|>~;`kZ|5rcdRhLQx}oxY6B%gSh^#C7WhsBf9$q!~6j))?ank6O99< zUgGz|aE*@ot|iiDb}Y^4uHa~|KFd4v$SX3Nb@cDOar$kK!`N$5Rc$8o+@coQW9Up| zEm1iKy=;vnoI-qG0xH#xJ62K2M(Q?X5Yt}5_oT|aOLU1LnqMeje3WrDV&5fbi^xUM zqK3352nxN}-pss}v_ekG9)|ChW+4AsqE(f|)*R-NVT8AxFS@vNDw$d&>#2)tdJ9B> zcaZ3*BtWejb!~Wg+s6bqpA;>{vM_`U^;OxR;{FOXn5dmAjJzyunE%Xh`#Phat5LeEM6i!W5cy~lg=iR3)E z&NU`R*P2&*(}nlJo!Q8vR^LZ4uSkcINWOPAk-Q7^Z;JXk4rUFseS1UzzKJ$+};pD<_h?}ITTTk z!SAhIoso6MFLqgKrJj>DHXYy~@izU!B@buFFKjY@k_G9r;+HBCwR-Jr@e;|d2E1lV zTu$h|C_Kr@izeq*yyAG+IHDKl5D~F_u{j&fHHG$}_L&IP#q^MPWkR9GKdp1DZNz!p zMAukXYrIx#B&_H_>5&uUBVxnd9aDux5=Gk!($W1A3lOG;J6eEx!J$h27Tatz8`f1i z{aF)pp-NGAOdzh0jf(R$%v45A@X~f)C}cOPX?!?-f`VGye!yXVSQXF6#`xpjOG!nj z$-k@c0|MU^lO=(l_Jb&hZMc!Mi^9Q&zn6&pe2bybX*hfR>kTdxJ zynKkFOT8n5h3C!tw{3W6$bj;$k>$9ygl+X*8dXx-OyH#43ebHls_!6MMIBIVDi=~I zKq6{or+2H3GNc%lXqz&%OkCwl9@6ZLDrBhQ=tN5 zZ@YnsF(`8yIMBtsq6~S16lziFV9!h>~H5-$?PGe=C zNrN}l+hN7xa9*7IqQ02y-8Gu?WY+#FQ;16F(_>-6VT6L6#99qz$V9-!3n=san#V4I z4Oy~v(R6SZi_XzS$Yw@Qoo*! zWb(GVGiRZ#-^;CZD-h;Ejj0gu0*3-d7yxU@5*&7rW*VTu{DCcKd)#*w#um}n0$71G zgC9^?Vw62_NS?8~0U)$~xIrK>OqIy(G600E{T1khRD*#(|V#`PaTi_sybo>wPgI-rS-SIgF8P2T_6HxSa)%gKnv?p~tgUZ^64e-==J5XLoD72+MyeM=*@51ftD@GDL zsSeBYD$c;#k!KWZ5WZ) z0KT`{6sHnH2GG@x?JunjoF`c{$B}CVfGYYY7?YtZR_3?Vf^@`~*%Wd_ zwEZHsW=9v)uy^hv(dMf{76(n0+hi&oYE&h4q_#5ZNIp1A&ReTCIe{@#I~!GoUASe^ zbl_TO_R)C?^q%#nzP_rlu^yEtnoPE`>b2rGLp)?2EV}`K3o3d2KChQ|HG(vv3a0M%ahG zVi^L^vO<|%xo%*X$T@e1N?K-*5#kzJ_cx;u=fXv!sA~+rusK02?rA@Q#SJjMGF7=A|GW8_X~Oage^(A22sJzC zKn2~mGHai41$7EskBGS1BHmgH8n>E9i&x8wc$x9sjywO{_$=WqjFmpQf_4?I z=hM;q^)hCMSKEv8d*wBy<^-?XTjOBM*QB60(g%LagBi}CXiHn5G}L%wiOh+ zWh6=!Y8`!L5_I0SlkK`=YKz%UFW>fGj>HR=O=odi0*9rjvymc!3CEQbSUer-l1_`Qey8~FI~w%*ykCrOh3vuujmHFiS0^7ugJ=D_ct zAgz&35vnLzvU*gmtP$LnMUcQ3Fo%5tU%{*mi`QUG-IJY$U(?nD!KUyKG8O&tl|7MO z|7biV=`)4da5pRkDt6OGjZ$pr-BXjff_P5A<{{}mq6Wwu1X-mTd-w$YI03|guJ`z8 zKM1XEPf`!ARSDAu5l!3xdK4&0V#6KLeUW7x^#RGyktfJU-)i7RX>v*BUQfOa-S5UO z^UztRK&!NO92I_*h9HWbMogQ(JFryX^7MwZ&qw`vdU&B7XQJ=FvOBz9HykP& zH%q;?WCUd>iy#*(Tzk+`jt<8~JDbqu=q z2Cb{BZiNguZbS7sczDt=qrf>6wT(OE1pG3|8^2}rL+_-ITkk1GOI>ZzxI0kVL7*9O z>r5-yM4ESTWN|*)oZ8Q(cGwEUFPOjFSWbMTQ#v2n00+e z^R3E(DWW(f8Z}IeyLB>AlXw%X!WwlNQxI)%75is#^-=a6m9O!ZyI$-}Cbr{bR?sq( z%ID`nmC}X>%tA$31zaQni+MxGqx+zNZftO=$7F(8-E(xcvhAz_V=pzNn~NS6rLHA+ zReAjcW}YxudFCJJl^qM^I1~rzXDL0%hK*OjsXx(gu!^iqa|i#kAnJj6xB_Crj063l2;Nix;ob;#TDYr{V?{k|^ZfgtMTZcj%tBo8{#)Dt>T6 zlZ`no7f&@%ItmboB4$R*$$OOdiTqYEF0wJmlJRdzctb7LG(7!)^OAM z)soY2Jqu+tSKkNWJsr3QPS~Bb?Q1S(>-!F>(g(RKy9lrt+|k#rB*@vFL#~;p1$#_d z^jn5kw0@NAk2}_|W8(HsrXz{Ts;%q7sA}D?oUZA*jcX}SaI2pC+Rj`hnrhA3(fw@a zX<1F>;)Aulk2HoRZFy6Y_g^U^D*7WUCf7(};_a2b5|a-2tnXI$a!t5Dr`^-HP-p(NL`QDtaPy3A0nDiEf#AGpEW$MeEm7X z!{;_8r2F7RtLM8{0O;?#LgBk`PfBf2&e4~ajND-#&7g(E5EQAARfRm(n>ZqH>GTg_x3I^>X{YUSIzJ>~+was4<|a8(6{rs{7N z=s`;1>g7y=wmQTSJtx^2dHPnHHg~5VQVTMU2STb&((;$O%9pBJO1M|FH^o$ZUw+?w3$KN-kR`7<61X7n z^)Ep_-3uba9WazGRl7TkZt{*+j9xpno=EEX>;(87U_v6fKnswaqk#nWFs21t>|qYK zo`2PY&-_MXEcO0W1LuNq=Vo-5Yh)Xi!)c7P9rme?EqHl)+-s(t%KVg)EXlIT@I0p|KOLGQ2b|PSdTKNhiSunK%Y6Yp8{uRQz@AROqBGF z8WxBM;)XZ9*a;#4`JI?h-!?vdM8XWygK{yjDfU1=muSbUvJ6grr4b6@ll~_UbzBr%xU$ z^RsqqvERf$R1jEAGypAy3^Gb=1+b-}--Qz4ig*0h-30V$j`g+}9%j4p9EY<`1Cb>M z9DL>yW`ryBlKr~m8+Y24s*5uM00wn*caGl&4(WY}wD}BFxu8aR^MK!9N_c&evKv

-69@ZOzPVEtVX zIu#zkCj_%4Yjk6QVW6S5gjH%A+e721T-=`DKD+9f>V7Nn4lq9)zhStyfOWzO;}fI# zuC|`lL^*0|U+rw^?rg?VYoia6j^c>LxsXbuq{!-bM;g@<8n}@G}u>y7`AOM4(Ov z$K1L;l_r7^z!Fy&nAo~d1FsBKqR<)2XMM9_Y}SFYZ88`+u#ue=!6?{pqQnjY#&qH@ zT0uCj46q0lLV;FT>6cg6t%{<((13c2vw$M|=r#-cb>@3qn}5#79F=vObr94#ZS~{* zhH%BJ2(prkU9%)Y$=$LF0#fG8(C3!&zw8yDu@tNYd;TL?JNIm$h*D}{839DP(!*I- zXM3Z9wULWow=6=Qn{E}vEI<&pE_3BRYK!;)3;--WBofFtx^NJyiG~#<}|s#*xObtY2I|{5Rrc^1=Q%a1$Ik zjv-1nl!?Lj)a02O#`OsP{b3LK!lGn_YjVn5OdJtVBU*Gllm(r%*`~(0OO5CX&0v#` z^eo0g{W2m|W~q?{%c)zbjW4$~%d(1H!rU>)QQrM+~4Bla&s(7cI`@{+6u?NK+zjAWD9`I(M5 zZjW)X2yLronD+K9O0&kKMfKyzBebL`Mw8hrlTJ1KcpeXNcaKTjq!{({Pz)A4BawC_@Rh4cE1i+_il7kT7h}x>a(-T4ANiLwQ@2 z(3e{Lf4bZ<*5xbF+D5_!K<7KxWZD)jFMXrt*$9~f11&(S(1mJoV*!LOF!}qNe4UBR zz!{6GMdSb%m41`2Wf%rBs%!@CWct1)xrTSGDr^oS{h6k(zJAy3hKKIj|IdQs3GYD zZO(1$I5MkY{)d{mF&R@s61J)+cv(T40$W3JX@Q|J`OJ+!WLZJbf`SHC(4x%b?B}Kd z6HHZ6`svT)FvXvTGys1tRcO^r*;k@}p1Rxuxpe8yYg+vI&=$9!=|7Ljp!`g6&M^Ad zA_fLn|60bGpiTHFkmwJ!a1PW~giAS;+p33M)(T~@uN!+qV>|h4cg>L7+}-Bhz}8FL zcGQjmcT8ZBwU8)-R)(f;z_W1ubJ5pEY>Ak-;So3W)VmIbC{(5t%A2aL?h)VFLM|61 zrEDT(^$O%V(nE=MV_d%4gFZ?rj#BqgC2RU~5kD$m%Z3!bbkx~0{@%+H`P8Wrr+Fxi zrw8UDy)gQ|Aq%L{y@NOfe`i{2|0|I2!AUyj?vmly-XH$R%Obo&i2nTZPjmk!0yd0q z;h6k6YGJPs1r2@uTMQF%WqXn0~{^GOp>=h99A-xUEvcn1a7;JGnwX z#Go$-*el8d{_}Gx@E_5MMJ}19=A#5R0&|9tYM zJBC_pFJwoaGJ}fk5q&=x0)IasP76L%?{dH*HgICBjG)j*?BE(jqjY1J0^xksA09tV z9KBxpd`+U0#t$g7Z=+i2(jbi@^i^O(Ex*n|g%uBzKn zvz($tGYXl-v9G>j$!9H;)NiI%2h2|{~tf(csS9qfnHm5BTp;6V^_5n9Y5b$i!kyCH3`&e_I993wvgPB_Bj=$#1ATm`Lm&CGJ`7dZnDka9H~%{?ktADDYHzS{x*UDhNdm8K0L zov(TehOiG@!T4(Cx{U@+>4^3?D#9f=<1FKs^~aL+ej&A6M35Up=7QnH4wlhWI((@v zIlv1aMx_g^m|(a(ozG-7L4@p$s=MjoU}RG(S!kk&xQv9HlyRt)gRSZLN`r?kWR;yifKZ?`T>#kFCA_Pg##JoDQJQf#yyqarfwA?`+_4U`g~& zT|H(EqB^AvkJfZgnhV1)x8r?7_9SKJ>*aG{spZ}$%jeboYwx^-xf}K`rNnr84{gaU zp$_Q|PqKNka&Py*P%806?fihbkeBmrBGW@U5%tQ?SF2=5WdnzOwnfYnN(?8fC&m|| zFoD^>Ne%JLiW5ih+tz3>*qE(sx&z*?mzjforwGCqcaAd=*kJ^?t3>!!9w8Ke!4iM& zIl@2cEVYXn2Mi+(uSvSHM7)&6lT((d)V`3yII#4K!m)FaU1uBp)2r&(RKwfud&s*z z%#S2brGhJiRx#h>)~HVQ3}G`b5rt&8sF1m+Qd~-vsMNI7^Qp#Xw+v1#N;dpM>kS<1 zj{YFHmnjxGh{rbX7-#^Nd$+6@UO==-dgV!(6RP1y+#8UZC``E!L7#Eli1qOY?Vc9aV2IC;m?76hVc7SG^EH#qocW0yY8pRERg}d6Fm4F;!nPj;; zTmv&VX!z%fUHTH+z~AzU^Ue2{cLuzQ&1dbe9PXl1uGhyyBcz`eUZCB_S$eUZz~HNR zHwQ>GBM+i*tvI7LujPE9ns2ckg6y<+PfyR-a?_oM(`GPB^>Xc&P5?>FEI-{c{5MyGRwNP_VgB_;(z!0=ml6RkURYYa96og=lCD@ zSu70O#&T|sKwsSNghOz!c?WFzrAT1#DloX~EGA9^O7aL`at)7BI1;T$w7%#HM-e_fvC9s*FnzY~_;2@LM?3tl+2J zxT}~vcm(|?8D|&go{I16Tdk&wq+?@k!ztyIXhSS$Grdr?mW)6#X`Hm_p|lVDLP)n> zfK5HP#Tc}rdo{rNL?maFbvI`RrgxXXUMo@9vY;%ER-OBRM$`(IcBwv{v(a2r!zFn( z94}&vmc`b?@2)7&O)LLhMLEH+MR_W`6)qd=Yj*!7yGmZW?oX;~Lf}qnE`3T1=L$0b zZwq9~fAr_w_4D%V?Q3J`@o4I@q=OTQY$(t|NTPc4Z!pzc<4t4Zd@iS@J#9k6|173Z z5S^Uer|jTbhkZ8x*c}#!U&c17!##er(+xr-^w-o#OnGNJRzq`gFQlt6?4`AZ@MoL2 zF;$1bv&p{c(Nr@qb-~Ac`Ic9$B-Yk7v~)XKJImLh?W$a#7DFwr-92pBTBRG71k|Mc zQ@zd1BagmS!lZNp%71@R(ZLXQHbGp+>1v~=ua@85n(@6bLnxJfaERTo;{S`a(jgd> z1>v<&T><%xD>&s(g;ARBWgbp@QY9BAhruoIz3cS!1~_7Kxh?N zX)u*=JI=7n{Z0{_iJ1_WlW(s7I39Va09p-iXP zT^nWp9?X*vxGcJPVAx&&8|VSX$rY+Ti_Y((6to6|Cz^pEi81r?y(q4gMXsqwsfpQ# z;`aJx>ljPLM=t3Vr5W4GEsII0O%8`@v~m2$5KBPuR%AE#zrwgToA?01HZ2S(P*O`n z=yum7L}YPLFk$!u);kwH~dDU<()4B}ePlU5Ss-_JX=R)jdktc~%7 z4;OD<)&BV>7@nKr0)xt+)zMasd>m)Sl0}ipT@nN%YuEzLGFSo!l&pPEJ?2^#PDTqD zL}m^rpk~j&n^pGA7!#oV>86zr^=o#+5uk*C`zge8?oU5$gFDv|azxU9nVe;}{ry3uVaN1O zB%YX(35uy6)&EUQ%7KFEIW$tx7JtYz?yfu3N5f$p1^?sGgHStcb5xhv=1V zhC_5H^Q^Y@_qpG*I3a6M+sB~7LlwFm1=zoytOH`Ly%RD62UcKI{!o*F-w!oqm-HV& z{RcHg^8ZlNH2!~46K9rvJZZ#3#wLf@zrcF_2}3PY=Ace# zKhUHy^lWO?twTuo5qHUnSEnC;V82ui~Ps+V}Di5&rlAJN~jHt(><{@b(u18zwwRDv2{fBepfw8xl7xXk=gh}NM$OzGkD#C zy8WdwxU{y)&=H6&4Bsr>(58Fwes)9J28gvFGX-+dC#8^9=@u&@Crz3rVqNuQY5T}2 z+n`KzUtq5(IA#Uy^b#zXsl&%xI}841VQv4=;JC48@yvY{65zD(!xfxaonqeS)*Ocb zt&GG>^ZMY9cHli{-Au29%~s-qIpKljjul&Of8~;zlSH*9a0Ha=pRoj#b(~7aO*>c9 zGICbZYlZ&hsE)`#t->iW_-zP7O(3>*X{JHlFi41^ujgt6#Ebq|Eyux;?;sWw-LUvQ z25h_DOsEL%Nxn#>L)~Kvr;MUvP$oSAOq-B{(CspsVjRMPU0}PUP7YB9@we4LUsnFa z*|k~99_RmWVZ_&fBNM6C=5#JF?%dup5YJt>dP zE8J>JHzW9d8bJ1(HS#n{dPm{lmMMImo!U~nCI#>H|KW?2^UjDWQ)}lBApJ{@E@V9c zPCgIJ51^X;jx+?6C@N8(jbjO(ZkUH~tJk`s2s>z}n%ucbta~qJ>@oIwW%dh;Lo4erdeES_JX}yTe00wQ~j5W>4YX9T&*OFatgO z$~P?YnhRet7a4U@LtVR1v`XACo=PzShQI9;MVg6N0-Nu_$RLV9M>s|`?q%Y|ID3c+ z_6v!5H&zzkhr==O-q2N?D=m6$cb_NkHxw}vC8^I({iUjy7@qn;W1(<*JNgcLYWyF@ zCUTOm<1>h+rHjBWF*ca(`f=RQqNN&uKC{xx0VQtfJ>1LWA35W9xO+Cd5lH=&r{9?B zbtvH(=Mh-pN*dvnUJcQ#V(bSlri*fQ#MiTMlXdwyt$4uFGjW!WGtO;2_ht%i?`?zH zaA9MgZTgUXJ`}aKQCF`84?Q^=Z1_a6)3ix!GkTohVt7Spur?v0Q z23{=nsItg8#?x|?EgK@svL?9?PF4WgM+{mYZwQJLB-JH!C0b(#^Tatf6K}IH{r>d7 z)eC%b%>D4Zopafv)}m`Sqsm))U?H~2mg$umM;9OWpSsMc5GfLk-ZdhiRV^Q>oEKq=-+VTtR;G{Z zYJ4l31}b{N#QAw&zki~L%5RRr;;+HFGC|9uxME2h!NLFGQ?NO|Tz`BDCs%Rck57Te zA%yzzDf7!eKBf16`4kx2fB6(Sc1K;bRas2Ix|+ZK1rS(Va!JGG<~y`kpTYvTZ+G7a zNu(qprf@-2?{Wg1(hjv5cj?_DKw{gA+OMXGV6p>NuUORWZMTm_+NmMtg`mM;e%7uu zbp8-s`Opp%Od8E$PLsgesVCBPA)5C;!8Fr|r5sXB(*DeV@dkfR}KE_N$_Vwq`zx+;=-x zd?4UjG4gAIM%+=4k^cjGZchKeUIr*`*yGX9aVlb*iqVz6_khB`5PmEMx@Dmxo4=_g z0pi2n?uoznEAN~jJZUsv0F;Fg!SHnkB0h}Z{@d#7raY!b?YgJEy+FX*?2r8d-ul0t z`?BIwl~;iK^kR>ydrS0uS2M5C0DOl-H_Yd*Z9K&s%L(c=g=@kD=H?8|h9LN%yD#j) z`kWBYXT4sa+==y;-CLGzi)WC(;wBoEkS7Fs>>G|wE=EF7Z zRa~38yH#?&ddfjUkFw#HuIb-phw9#LDN+CFhE}9*uGdtw%No?SSiLwOlmlUbryv3M zl=b4vzh_%2M}>n?|Ed%p9S;N2D*{bHG|npe*`z}J;&JplUxR~iBAVAGYmB?DnM&kZ z^<>C1K_NT)Xsl;=KpzG|o1wcKW$gq|gcM+v^Fs`Z4y|j{Cr(4V%c}v$hk(WU1-G$Z zP9JX`No0vqCUkD9#=rWO7t_oZ78O;JlT$yW^Yy2B|1W01LaigGoHqQ6PN0=gVHysM zOdUL-f$ba6UHiZALt(|QHZFcs$X;Z{Vm77R+_c7W!}6I5zrk+5qI~D|Wcsg1CxV*{QC+ z?$OrL-QL`1B??ADu12H~-!6Wylap~>mob~gSx!wJTU$B5kJ;hL^x^Jle@TpUczm6( zfp0n7+yX=A%Q*d9+eo^#!LMXQC}lhD5=Np$=g+E7V=TVbEWa)zrHea4U|DuCF@}$kP95o_*med(+?a!ys`zNH2&MLrZZEVm4yX!}s(1Ka=%2A)m4| zCjv59B4=2ZcQN*eteP(|gEy3)A3QpDsf^GL3B2A$pPg3GMt_^mRfqgu@|9Yc*BSw$Y^;SnU!7z_zN^y^ z3-+HpW^4es^maxoUC+up*GHOGK(pUtOk!>wX-JCdPGM~3({@-!qkE;84~iOc>SN*1 zwu%Ou!aPjxoJAmWam!5pr``0UILLOss*0 zw;ca#;;kDkjW$zVr?&ii#ga&#%@1JZ$H&C}&N4`n^)=r2l9H@i7PdqUP?1+_fmc=4 zt@f^f9j6R#TvVWAgs5Hl5n@AWav!1P|G)LKccYolTAYUfve`{q~+&q+vGdkh1dUZW&Y&H&ff>0h&2n z&FL?b4sg1NopHKk>oQE4AdYpJ4%HLyDJq9O!0bx}E!c1dK<|JaCN#%D*zcug^lRF?Oql|UHa>hJJBrL$rF^38nW?AeQuDbExWgR{8M*_JpT;#gDSSXtt1*2W|} zV&abViR>@q5f!pTZDGEVsSUdxr)|OlWDN;^hUlW$%6LMt*LciWRzowwh^ru=ggwO3Bv`qAGh+2L~ZXX3W> ztoI2Po!+WXTUnh3HuOW|#7MSohfXMxr2IhudTzr$AnAi^h{79s zz)2nCDgaeLQbf&jogkEsGW z`!)^?dl-x4hsL>8QY@?`CirZQt>a0(exIQ0f-XXkR%)?F&frawK_2VXKDpieYmaWl z`tME`jx}}Bj)wLBJ^MGjWBFJ1AI=CRo{zv*C0VObHXsTKk)7PYdkdKPjJp><0)`8O z8R9#L zV7LN?F~m%06|jZRcr2o5wD^<1IepE%85U2mV1YwcmsW5(C~c)5Cw9W<(`zdJX;7I6 zIV%zV9*?zS7A^}NyPg-oIucpC;)F{6ui)QM{~7&qi-%13r$_o);pdT=kGug zgWIuelUm-+z%u|f#At=l+Q-_swCo0+Rb72gfd<3%3fvHr?q zrTlYZJH)|sT*+D3E9pW}>&ndOr7)w*sMMCh13^k57iZaObW!2Z`vH?xzX$n~DvbZq z)2`>^)8+*ydia(+Ynt9sHs@B)O-K+eP7iJF#{}4iw##oiK*fo@-BxLQ_d%ed87FSV~J2Rr; z#c?qu6sgc-@*?Dy9YCSGmSl9kw`?7G45`V+z>IwsV@9z&)|yS+qYZ&4SP|(g1%ZPD zsB+DU@!Y}c@tHu=u@y5@-<1b$U|H?`LbxCz@nQTb)+qU^&-r&6oi-dnIhra7)grMh z1&CLMR#3{>v;K+0$4Cbn-}!d|!P5&b2^st_+c3Y#9DzO(@Y+-NpluN=|CNSIG*2(> zU=GvnQl&*x6dVmq@Giv+ziGAA=s4?Ge`pj8Gg*fYhc7AB5L4pW1SkUJ+d2$!Xjg>{ zz5M5~%eLz8v$SII?|WihIiPu8Ld4!a59Ia>wK}P~UD*Tt?95MZAD;xT=<(}|Tdejt z%0le_hp~46j;w9lc9TqObK*>riEZ1qZQHhOb7GqlOl;e>lZkfEgZF#C|KGc6SFP$) zt*%a2S9jIA*L5DpdB>wP?$qvhgR~9yeM5)3s$z3HD%eJj@IMmnE%Uq%NWof3kv@ao zhK>1A$ISrqW1peCuFk<#;%lIC0QJzVz6K|#N2HL6wL|6fUvqSs+BsDGS+&8;$tk7_ zAyyeDiT?Auy?|=>pqt3CsTRw<4 zPg0c#N6Mg>y&%HHbS5&OAYp#9HNt6FMPYtSzKKQFNoofkLd_?=A8pTZjIeq^9za|Z zw?MN34mRo4hn#13->)cI#b#}dh>>KdeUIlxGp2Q0xyeS(GmiMz76|E-If))tsgb@p z2)aAPU7qvw$ri=&Q`j6u>te?3$1!u*80MF*~PR9&TCs)m4T4@VK7VG9=Pj`KzO}Go=A6 zH{7~egJ*7IS_mG`>{V{4(&DF!*W=61>FJy*OKjb7jd}c-dAKHRBP<3E7(9y#+!mC- zKS=&zWZ<-i^EeY^WiFqdIge0@;dWsB>Ye{DD=5V=L+s{H+W(t2k;=-!L6#7%6)*+BroijSf^)&^o_-DLB#sA&4O$&~^di(& z(knG^$F`9+7RWLBRg0q2aS^l07*lKP>XN~RH?5JLXdB#IvC8I7X~nwwd;E4VX|Kc! zuEbI7{KBfL-*kXEhAb2VjysMPU&8684)Xb&B0L9|-9wX60#gn1{tgw5pF7JqQS?C7 z(1W>qa{ghSMgIKx&5k3NAowEy)KH%Ix^}#9@tl0Zrk_1)IWR*ExEo$sXAlz1zkF}w z2y~dR!psTs1rY$C5#n>7sYrEdJ6D33S5+(XJ?wXq`*v<*V}hZ_-_-#fQW}8`Uq-0= z3rO*620s1xQ5VNTVO7b`O>Se=n2p*B@=Y+FGRL{aaOnqIyFM_1zU*;%Gzf$+fgc{a zS@i-%NVI`@vm=aL$)Ot8fn}T}Xp=%l#FNBBCWkycH!F!GhDSRz?3Sqx#-(mjOGS4! z2?}DRgW9cB3%AN1ONZ)Sbu1Dpl_de9xJ+a7>mY+TN)#-VX}*bt7W!HpyAHu)<@eK0 z_Fb*}3o#BwydU$w7s>-;PrE+8bU8hn;1tH5PDs4nvr7`6_a$0>6@5q<#Ar2-*O~kX zVZ{-4j@^g5`Ls>u)OBw=-7fT~LAJekyGUb;Ii-fVTy;lapd0oP^pD_mr(RI@dFF5r zk>{-?oidb>aeV{3npF_YIE~HnCvsPqmA(7~$hk2eklml{ve;{1OGO>#v#AmXM~u#M z-(FwugNh!2t@mS;{IIMZudZc*O7pFA-aP%u5m|CmBkg}XAol_>>N$>{@5HnRe&rA4 z!iOho!YMLOY#dcs;n9wFyADxeF=@uGTBjh2ba%G|3>zIY6kU1Kn8sAeZ6{{$;|}KoOHNly#*UsO4bY7DU-M3{Z6>7-8(^|%}o)Hwp9mRuA;eLZFt|$=hO63|i``tzfdAK<*Ul(J21_3{qp`%+@g5yDd z9qn~(YyMWXE>Q)dr6fdAhtWWS>^&V9PN-6PIBm{#dP8RT+zw9NZwI6!14Buslj_)AD|+*?!r4_^ zdv9A~9p8Zn$-cD7^A4mXzy{A;LpPHH1eq90J1t)ihdCv_lB}ZGqy(Mk?I5C^z|o5{~Y)q9`-*W=U5fX%3cf|F^)&MZQwMdb7dS+ z8^?san4PS%00azhY-2C8RnJ)TBf-z7v6>FmS=w4yw7|?bc!oFSk!$X?;vs_m!s(lQ zJPy7)arC64MumNFtm>}{A)5IXX89q`@Y4Nnr|N=vHvm$XcT{#y6p6xpNjg4}^ylEN zgtWI2d(5R;^WLhCVo;g+D)AHmnyjN91Ei@Qly&2xs#n#SRvn;45OyWb00`61eyR#Ni#E~c4YzkDe3!L-swAzF_ z>5{nvtFM)B7$8joE87a@ox-usx%9qNuBn7nQfIR93}NSQtpc0X1xzpM_Bej8yZ5Ua z1pq+ zui1KI?aBPUD}O=$XPJxY&hL{-0L;HqZn&Y>=Y@0BqT}O9|Dwu=>|l+AdrGnN*;ngZ z9yusv36rpet%r{e8%xml^+>?z+NQe?AaD?sKb;PnUs^TefKd%{)ZA8FNXB1caEr2D zx*|6htX+MzYHHeIZ+nN-Uj6C}FQ{O+BZ9t@BHtXXKiDIC6U zQiH$MrtX$dPJ_wJZS1IcwW9d~Y=LJdX?~3*{Zh~ylgs`i7}V%oORv<5p@bUwFIG-| z_#|H=)FY}EywOzxwvFB}xI;u*P#3+?6O>o z2{WQfJF8?&Y4|&LIz|oK1N%uq{_ki|>t&ry=%zO`Ep-X??$%ifSr#uDfdr19(4FT^ zAOm)q5YJTBlc5Bh$II3}42`+(wMAeMh1P{v@`uMauy=etxP5!% z4W*0pGSb~dQ(v-32&AFViT9Cj$inqT;T3Ya=e&{+!n4YlcsJNmMfKZ%wJXV{Xqc-E z6Q@XIMnJxuGfFU@%{aQGT>i0A9{(bLRuCoTdHN;eJFjlgB%;7vq|4Tx51W>z$!pbv zu}*OHIijLqj31}r@#we@Ic7!zZ9!vNk_~eJJWiqkj)$XM#3YrN4iK#OQAlqRNPWT2 zU$GK`$AVZw*Nz;6uXt zZp}^HHWmAo&-Tk`v{gDx@>$YP!TNc@KCam6%7mej0cCIqyhgR1@uedT*!i@Fq^U=JgV%aiBW&{?A95m@ zb5&E21n)>mNV{Oo5{Le?Gq3QJsZIk<>X@5$sNn`s2~k9T6#L$F#IhUMO_mZUufn72L0N9h`bNxHZTR4~J_f zg67K;D*~aj;m*$ZjTc2jF*0EgI8mlUg+7{_k~7hYq7Vu;972&YFn6%XF%uXq)yuNBD(4>%HS zG-n~4DcRMf*oEO1H7@m(+5fCK=xdzPRM~e`*#M09Mj4pv19_dKO?S}jg2s;qHbWYx( zW3UrVeSC^Ykn!fg{q||BM4Y-NY@Or9&8b*+4B@`(1?E`=%7c+_0ZwO6r))8Ix&eE< zl$^$hP&oiL5XUi>Wve>JfSpaUWG_j`W&bw+7RNBrr){^1>z7rpUXkJ?8n$=H3+eR5 zuxL?kUW^Gx*NVBcBJ#z)L8{~X6 z?P_vu>3)STDP_|YiwwW+T?{~swDXk>7XslHkUtLYb-=9@#mR5O!O`Vik1Qh^B5>=cyOvs0tdoy{wKAv}f4~8~ zl84^pri2dM#s#f`WofpaxaWrSM<5gl7LcumB3Hk><%;-QD@uwxfkumO0eJf0y66$Eq~*bD$V4mj%o9S0P`LnGQT6OL@?e6*%1gqa;|@FNUjT@K28 z2^)B6@%n9qwQysi_1P$up6nq_&iU6ujV!Cjc!h~Y{&4~1!9-2vF3YgCHD{JtzOoswOt<@XRjgH=kD_7s+S?J;}j zv1u#S2FDO4D(C&(a9r62<{?Jw*CS)uV~D@B9|sSj0f%Vg4d`B#AwlYruG`|m)Sx|z zusYhyF&I3^6n!)D4mp|zJ!TbsO@Q>H7$a~1JWphdG5WXkBcjh3loK*OH`ReeH6;-} zR?w_Ey>Oudby@?u#Aw?2LB9q)b@ZQnDCq70({j3Dh=|ObGe1C91UEvaaU(A7p=cel zI6ZANO{PJW!T|8l&P3W_Xw12I_atzv&6)66b4c#tO6Xoq4UbJp;*v9LA{A!;jk02w|cx$=;kMJXXhAguO z=niPo%vhW>24Bi-DSJG#i1(!8R3p&F6_&KYURv~vexry_@U9RiV#}ezFh~qTJw_=C z%T^=RO7UP#aG_H?wSYwD>kBvgTlk4obd-QbpDIp4uLKA`$bkUihlW`VApAJk0fe7o z2$+9_pPp)7wSGyswl36=u=j^}qFH}J+V6g%{s4AuCiXY#eqx5}PKpv0P!M*NGjt1& zol|IkMVA^-S5})!>2XZT;QCIiQLOWNCO4!Y?i225tF3d^7EPKCc)Ui(JEM-(EUO!m zY*HvFHR`NK{6EG&%pc>QRanKBt{f{_!;uj?FqyAtc6PnLTVOrS<$CZ@F2S3{s?Fpg zU%h?qZY<1UQy9y4My4HJ-JIG&JgNPkwCA#A)h|I}rd|H=Pt+6kI0g;}hyIuGKSz}{ zw5MLKWmwnKi?8M0`ab$C=?isopJb&=%ned51%jhPRkc>1slv6`cPEWM+0Ow>W6+-( zPuS;ib?|tJv5jXW$;sU@d#KfbwLrJRi{2wTai0O0k0YbL(PUc6JK46O5E9)^CBJd|E;Xe~ zVFItOWNq0? zXw3gfpt?N!9+yrTDn*_d^PckbYHAjPAFLh`M?7Yn+Owhq7rqg8MZ~6haky_gm{X1X zR&cigch>YvnD86oU}o^8+CnuSv|uLQ=ctUx6#b8y@^mmwa2LkLsA8v3rlCtqUQwPD zy@V*y=1~kN1VFX@s}NvcxIp(m{FwJFw^R0nZtbN^g-cPCWx4K$Ow^1Qt~*P_bsEx& zytr(aVapE#EopfQgWHZu@fMrH-@;p7Kj*K!DmgQKOc8qm%`dy^IMJM0Dc>jO{|d35 z6V6}$2PaeX+_|wwn<_#iNSmGRoyJ@U6DV#6PZotQElZU(ZoOOu)weJJxdN1jJe_OdxC7sKEaV zG$Yy^PEMziggJc1e$5U#VACQ60uX1b`v1+FA+7Z<0q#$IwR;>J@4DY_pL%v&*|3KS zoj8W3xxIh=4lxFsuZ|vr2GxAA>AM)iBwmIxc<4cOO`oW@%X8(73C8NNK!~B zBk6lFXcNUK5^8JsgCJ!t2B`1$YmDdvYSCLcac^9HW1JIF)v*~MrM50z%6^8pKzsHd&oOD&ZG{7&v0Op zmsJse$_qg_5M%9&&?_mw`q*F5Zr@D1zGa;~;j4)OdFw_v^&CGTH(xib=Z=L3a4iSB zNt>$ylh~e0yaBpvOLr7H>!z)^M`|8ay!Osox-gE=l9M&1R1A<-HY0{yXs{{s)y^Z7 zeRXB_ggd3WQUhctq2qeKQQ;U~H@YR{Y=pS)o%OsrD=XQT$cN|5f~#!~atJE#d2b`2UM>xteWn zHE3%vk;43?h=3?Ng7CSa1vTS>OwcK#1_94oc^4M5hSS_=!SENY@i|+A!fq%7H^RYT z#^{&jM_WxaFV1`mre(+pzBfe4q4vF=8!edDwJ1MS0C5_di-rW5qgT?KW;KJiU4gW; z|0~!UQ6cn!kv9G{43O#PgBr4Ua?|l>qk@BKYFD?e3sl?)F_^U@GdoNYjFsRp%ggnF z80o0>)Cf-zzmYsj3l#*BI1eSwqTMO|7&5Bc831rwWo^hWZiu$b%A!1WS~wv6F&a-p zmNV>fd8sA{+)SE}lS#9Tj-@*32_8lg$IBR2$nTh)-h+!D_|$=D-Ao%x=MAF~ET##aJb zG`KV<+?ZWrXHGekCe{m+?x&i0!V-f{+2IxMey9e|--fg!j6^Zimc|AN?GE>ToOx^a?8M z_OcreU{;Xp0?dk(8EIxl<3BxH>Qn#;oh5-hi*F?m06m?20Z8aN@P8z96<8Dt)^0(F zn-mRb6L8>1j_5yLMFhTl(|>st8w^Zq3+GvcaFV!f<@y{sCYCX#t1h|cZTQWBJ_N}d ztEZS-J4Rn;6L;ne_;*3JP$m^N4LP=YRDXpt`Et1gv8rWC80~ht9&(f$Dx}>s1|g{_e@Mbt}|;@p|m~Ol5cM z@~fnZ^R#GXm&E~=WS9au_ufV^=j>bi*}2V)>WBs`67es@Cr2BQ5nRSXoYl>q;MyD4 z$;ZPILL`pA9Q;2Iu_(KKxMuwUo|N{#eNPJe13bZR{SWXoS@^$!CwbKW6L^}n6-$f` zGS`Q@5R&}xW>xfiCP@-;+WYTC?QBp&rt0pxMx;PywO^gIal5NeRiy#0?83eVip)#nU?2m+T_Jd}u9LQp{d351W z{zf=c>h~{$!mRO%-L`qy%64^+(Sg;h0!I-Q|5{6vMf8FXhQ@!*^&Xi813Pj?lyVRP zg_&kev+aCADy1g&0RcviV6{dn^Rb4!w*HQ#ChO+b(Dr7|`KGSH2n=~;RwgnL^?GW) zGO*&>IPsZX9Pt{hS)nO6mK~HKe>@9L@#+huDek{PVOhiBt$#UTj;~vlXZLRsV>I7< zqvlFKZ)gTJE;>Hho#fq!zY&u=>eSHAep8BGE(RBjx4yCP_)cZoHPczsjuiZ@VFGkE zgTe{Mnx_%EJQGA4P3o@nqMXhwS5P%LzoKtiS^v0vI=e5wLV%kZa4HAG>A0NENqKt~ z@L*PAb6id!(m$=758$yX0suUgYA{;ww!3bZUiN_&r^9i1tt(37)=Nk=G~k(yL-xgL zXtQQa6bxdqJ$4l|G6J`N6CI~FAA5ugw2(*WheX&5w3yoxFS8oDM%Cr?-TbaT^n(rC zVNT_3?sK@dZV4@`u$lSOdYn+0+nsYL+7)1*I1l*9Y8rvcHGOZl|C&G7g$1CP4BO*uWP$oyC1Il7_uZ5FGn9 z7Yp|`2XO)mOdfE?1nTYNl7Y{0{H4cxGd!LaqMR=@wz;+K-ND`GHCJ#zn5UlhDx#Sd zL!#%H@_)w9pss($&q9R%jGw>y?c>;&WM@Dv2c{P>sNWIH@cz+KXiEP}OTpvVKDGuw zgPmdO`tibT6O|7;=p$gM0!+mzv=|q-J;|1^ic-VO_f2=8IsSX+t)s)(q&OApl4g3_ z?JYfYXmQjnATmnc=7%X{MoGeL^y%AMaVbWS+%(!WCCn>IM+Bbt+)waEgy(G4~O)!aP% zn>ld`r)vo|A!wkz`+n`%SWzwO?z4FWdiw8a=0~L?pGC5CU-&{!)+GQ>@LSc|RFSgG>OP2T5W`CBWVJI!oAkPvzwcG*sg1DG6DjdTFXldnrq-&xJ1=PNVLnSkAulA2aH5>s;%z3vg*N3 zGv=E8POj#)3&&$hXgmey`o#tpfoov{wGcA3-s=a;)aBGXJ<#PQN{_Mg2CRyH$bFG@v1;v3D#}kdvXuwW5mon-;s@*g z(Glx;C{}RCv1)wBo9nS&{;hE7TuC0IUoi4V{-OoB|54=q5kW+j;-{7yE{ zgrbNwXa4w4xP2p@A4Mz$`bR?S?jupA*^5i!gff3jr=zaS&o&$BxiV>|_`jN5q{mmR zl-8xBxRH^5vp`coHkKih#}Md%>Q^lL$DLcydj3wuttcwT6Z$XaH-nQR{YjkvALduR zR(bGZWA@WU=1%i!xum|K0`0P{z{TIm`GacC;UybjBBr|%F*gQV8vmhwH{<}+FLZ%j z`OY8eH*>z205Hm-E{>#KHQHuM88|c|1B`M^zApkS#OB>Y)a(Xym>}&2j26U))Iqhp z$$G3z7hcBvI=5j-D=HwN(`r|kQ#LEFemPg~9~~c`4Nuh*_Ityt+fR3t6r^9zE>XTj zkhncqc&l_zu<2;tQm$;+%UA`0knp!H7E?%TTUB)3ZDA0c37`Wb5&)MQrE>n7qXrHq zf!c>lfBE`ajsNcfqcc=@#gW@@xaF)~9pk-p(8~F!y>+y^?E3!ip&i@wiscICHZ#Yx6d1_q;jzd_2D4hpxGFJy%H?Nc4PgB4nlsI2MpFDCYf$ zk#;qKu4xS9d{4`9$UFHCSe50woa;!?68~PoG-C=$i6tAD>L?-Iim*x;e6Zr!VI0DG zzm{iKQQ&bs-HZGY?uzhEjYn--SSoDzv$W8jM<8+J$8$~S8mp|&^t?nnYYq}Si!RN_ zS6bfB#}?meo6pDB9{jeA$H;VR?b$=uU0B`o48pHrU@aHDD@y@qzZe)FbjNQj%A=3+ zdwrU*9w(&r#z8KY`k(TtbxCE;s5Zf-J;J%pF4>sR=q~94RF-~7 zJF@h#v$@3T!W*N7Bb;vi1@jxQ^-c`l%zNuuaLUC{K1|>;Oy{z72W4zPZ@u7FAze-# z%JRp}X4d=KUe}Ase*R`=4xu!V#@KODHyQ(n zw?>&}{#OPX!T>G5P*+!09vH8pt?fMDXy&eZgys(Y5Vq}Gl8=sb4=kN>r46kc!dCa|XX+RFL&9A7fgH?>ytIYp;7RL>R-)qf^3%q85j;XDIBY zANGepG+a5?MJdS~sXPkMOB@ieR29K67=Jq&LdBNg48dpOjI}q+bOgNNMvcI(>^S%- z+luwctL#G~cpS{IRUPm#OwPBlhiR?uFv#GrqIqD@Wv#DxWKJ*h3il}1dyh+arF#b5 zt<29qns*SHbtbc?>%A4HiPKn9^&>Q&X^SGV^k}r?xc|6r)%9!Tk-P2y?ys`F(XI3< zVqUbo=8Pn2^H9Yu7b`)3oR`#~&cIqOT)Vm70ZAgAP+d7D)}+l`;6>2YVHBet6`a2o z0|!ZW30;Y$fqXmBJ~sUhtPokv{c`hJP&14@Re5lMi{*V@`4Tw~t_{@d;4_xbxn?x&ZxkN4xy%9^Vw?Pk~J%o>h1nYz`4uMIYJPx9D+#@mtEUX;x?$45;Md&+Ce!;cARSnmVXplQ%p9ve~}euL7j zI56mnGLHHz&juRYKtkAeItQQ#Zw$VDc7g~w=svb6FS2mPbMUkJGUts2kpepMlq}zA-u&pYq-hrA<$2ZzLPC zOe?$wMp$9l19kS0T1lKh;PRx30WGQ?sRu~S-n8KSm!LqaapFR99z)b-inKlAOnRS%YTj$S|^d7CE#5@SrsD|gzaV$MmwFpTS=)XE>F;+dW zPlnAG{d$c0+DNJO8}x`HCtTKT`6#hXE77Y!M0}ZV$}a-2k3eT{o|6twWR3AbPN=BM zv}t0yOPptpiSYp6R30K#oF}qQz%1WX14{tkH_?_SVtRA<&N&G}p+FeyM~&p=FOq?r z_Cfg-Osy3pkleiuJ6Doixdl*|kv2*T_|+m$Kb@@QF>B-p(eFdvRpwUibJ%cTsvH6Y zm6sBs6uO#)zeLRjis-P-HMAG1oEWsVYN+(VKHy(UBL+=6spSQx2ZLWU_>YqYqDR^H z8|X>XcCz3}@b>co9dpJx;7UxTFR}%m0o`p=A67jh1|O$5R?en)8buud>CnTjSl4SV~k5X-DR#BxYf6vnDJ?^ zPRtlA?G;kEs==01nacu~agrJm$#{RTi|9Vu&KEqtZ+$);ocVV0__| z`r6^v#har0c9^9;^B?nLt`Le5WdR?Ho&<`$H6aeJjICVYzI>kCQYW^sO*!~=QVr8f zIUa9Uy;aFx6v^`VzQP|jy-|ZWdI`#`y?v-E5Pv_&RSV119Nsi&5x0{QR-oxwF8=xG zogiEs4)`)_GoF@wz@mT7Z}5i0FlPE>PV`U#MmJ7vJQ5IjXY;PDjK8@wC-B`IIhv84 z6fJDMb!e_~va=A~*}!reIJbj87tR0y9XH%?-0OB&e0Pj)GvqvV^bK#gVM67FCl0#D1EPLjrFn9zWS zhPfy3n)I*KYCbT#(kE=?KQpT-%J_44d}~Z3xjbI5CPw6EYdf&-!M!th$$!qEk>0|^ ze_47p|LaU_<~Xx9ATzO>K6398o5`M>SR`RPzUwT2^2^|nj$X`@weQZ55>Xv{sYb1N zRh&12%M`yc_G0~Qnm;9S^-B?M1r;u!Kj;1B+2uZ@e!7D z@NBfa==#S;Xqf0DVK+*Z^3p@RghrsSHLupYy?WRp@6gy(vHiWv=omO2o5dQT!YtvD zU<6t^;t-vFs8f^LqzwY0&2@{Q-P31J=-q6nQ|4zbguRP&d!8UiNJ&vZf~2V$(}=r~ ztWQ!uwY_5&FgIlzSCHV1(s6RHRePQz-oOZ*)o%pb=&K}7f=?XZ!8cO%kcpEW>UCNC zEx+UZs#!AL>7^Gd+zh!SNN^jQk&Z?k8!LrDN_7$Emq*vV^ESA5Dn2$)e*{Aylr7)5 zH8r04qm!p2mf%UFys(_j)E)~TAX4Vbg>M}TeTS-Z$SK-D%cld)_@wa=>*S-rummWE z%E-0Kmx04T%?veVf~be~+ad%`0#d`FEMeS2Dw1@Q{ZzM%X=CelwyNSJ9P(j9jdKV0 zd5CcOC=+H7f##ZXhd9Nu_Qby-!MJY^yw?QF5?6c2=s>3Z70ULRDF(}Wp6_q^OJkSt zX=r$FGa{_W=y-|79k&4cs2B)2uA*QBut_;0?vX!oZH>h5QFwnQF1l(kt?NC#>Ai(W z8w4P8xpPijyGAf*zR*ryV~y(MLET8wdk46yZ{5Bv3_iRJ4xOc9T=8mkcefXK_onY@ zgwLiV1P8HeB0KV^Q@2eRP$Zg>>ovK3T<)GPJVZ9QnjX>W)~vAm-aX>y(v(Sh`l`^dpoLk-YWKTf9j=fGlHUY^ z#}Ch`BfA?R6@zx9yRG%y%2ci)=dN)Goj4TSoDwNV$y1}C zy2K;Gi`hUc2L>-)D9i3Vg{+{KqxX3QbYuOP;kh_RiTlFbWdi7MEG#7L1AX2Jlp!1; z>W!Gv$U+^C)8a#*FCO2W99j97A?(lDfC}bWex$++YCf~ycQV279S0-wrQwG(>q(#& zq;MD1<8G5-%~-|Y`msq6h$)guSf}yr=NHx#y(_T|Pt=n~^X&@BAH7Jw&W}Decym27cGcS2D0+bUT8S28(~ZU({)%Evf*hs zaojc?i2kk~=P#ri)c3+)7R0~D%qxd31m7DMclnRIr}=#%1pNB5juQ!ZqB5&;)hGw) zVXaY#{NDYR9n|~0_7uqaT>{CUv~UInK0v0MIdh`AP4p4j2F{1A;*4BC^Fpot9M-$sx+pp_){@=#G0B9d_aBqkb;w@Iy(5 z+hGUkjb_xC)m=hC%P&_;g^;`v*Mb;3cmm&Fbj7q_l0Yy4TS%q{j25Eou#eyN!trIu z-!VET31Iba{v2S3CPw`R1To+3i~>nCG{tuw8iUi36RGdg8Teh7GNA2Bqzobp5(Q^n z_6{%IDkpVKN#2bRgifXi8p}e-ho)s0^TI=lLR=&Ec+vU9kFI4vZC87efdLBZ@5I2x$L<}_jH4FA^#)EzUa ze|Fn2l;4HkF43SqcpUCgFI0V$7ceTnE;Zvj*`oSTdr;x10n+<{68fApHth!=3UsOC zv5t~VQAYN(2ZTY~OlEuxnmhNDj=-dvl_?Tm96I!0Yx^X;xX*m(t zNUm7!f*D*Kz9oF}ahe^VC-v(X+2TlYD>3Q{0zXsLo zb&H(yXL%0DI}lRmP8ot0j8FPYYn2d0*@bOfACu!oHx(8Rka^5|$4Wf$vpx`EqDaye zn-{V95^P|AA+`-<+TCScCk*VPwhLkEnUS@@Y#^3|qLF-+y;2!NXkf3XnLgZjtXO9# z`Te|+)-2e~3*m#PNdtR9@c2IMXgF*&+n8yWy)m z=Bv$ila15yOA3Am@XEQ6#Y=UKckbEct`pzuTfws2Dr@^bHJUXL+a$E`n+H$ndw%Y0 z7q`dg{`JY+j2k_+Y@~Q0XsKCKZ{h7zVYUb6LBq?j3}1*s`eaHW?2+MZS7pJTUN+pX zHdH)r`MsWtw7T=)U41<1A$aRCd#$Z#z3t$-qsH>Q;T1*x3|SLMl!BlzO-OAiCXO2n z{sdFR=V8~rd8qJ?>Xi37_VlTm6h;Gs#0eH6@cSMWH*wbmY7LD-0mi7Xz2=N*X|d4h z!<94LxxhBkpWJI%kD&{M2E0C;370>ZoWl$EQGTAp;|4nv<2DsHwlowqmkms=&YC_E z_R-t#X@T0Mu>pY(EJeO?fp=fEiW`Xfn^L|!tu4U5yh#0Se|yKZ=b9J}B_o@J$ww36 z;pT2WdZoLKL=lxqIa7B*U8_b2Cp(XG#dMHeOt~D%u)7-U?;6Z6X|lX@U-}x=7hYdT z^^*rHOaZ$WvK1V0NPX98iHgv+?~yO-w^4^kIKr_qx1T!Zs{TIVZm08V^@V@<#qLoMN{i z@csKUHo0#uF-#06T3MEw-sNW~M_ zfIM|DgX7PP{JrO`MK@TAY|b3;yNJv9TiofDEz)5E7_2Z)XRLi<ie7E@IFroF!HiJTNyx99ad_;q5XMhN9hU zkz|nhf=C|eVid!Y>#dtBaGlo*xokx@7D;^Otsm$)P~*8sLo|d0S$_>a=HmyGLT53_ zaPKxDhYkV{j3`u3(U5bGSLI>!QpJ3U>rjW(O9&!x8BQI0n-&DV)#8+n$v9quTES7w1?k|TSR6>e45CrGUZHI-Wj64bh3qzfz}Xvd4R{5SoIn>o z_&c@uQ$6BZe(qP|uuNIxQ1g9(-GIZ!k(SuB_+5{I98CCd{6u;-k{VXXy}bv_zD6lO~o5D;zAA^2Yj zv%g?~)|Cc@HfT#%wrah~Td+QSF)amrSRQ#!s0kO+=Hiqu7&C0*d~E#i)Gd<+5G4Fr zAW^kvp#vpCHmts{Kh^<1pV2CwLq6`7LI=jY^@Q~Yht=E(M*<{0I!~dT%qwvC5FvU~ z89l{8skK9Pmf>(!gm@I9wbt-)^eW#_n z@QRi^?bl<4(;4DP1B6v;qDmRyNM2kTj7}iAqw3p7<^m|nR=eY#JYqy-*Kw&P>MoVe zXFi+zoQIB{F|c#MzOTokO(O(8N|_Jz5ZKpT=p9(oHp@@Qy9#Z+;$dF{-QU6}+Onwz zt0nL4F;TN2i4F3ni_j3_Y12j0aXdG=*X297NAFzbt)+KGH&pN)^V%oP?;&Wcvf6k-$mH> zFP1hkrB+5UE*Lp<2+hbTlqa6LM0gdBT%alw@au;zT>My+saT7bs4kD2%?tf$k2`gH zL2z#{6uHqGj(<*t={SsVK{f2GxiNX-fvH3XGUzSc(oga`(`G2fm=Os6jBkDo_m@i} zH$n~^1UJI+R2R9u0HS*wi(O0@QZ6#!ifGU0_LJVB1M3F*NPnZK@#G3UcgcPa@;9&@ z2rrqgz`B#uC!@p#+%Hq3o=PUt&wVyUxg=>GYfl9?`f)$TfYUGUiX ztr_%wcHLfJ2@J{IB!ZRlyv|ky^LO}Yo5bV%*1opKkzx(~p9nV1tLee@*m7?&BRCW!giV@xDA(2m^zy|b1XzDKH&VP%ak^<}B z+=#suo^y>(h9+b)xT1&{9TnaTp&B3g*eP!?Pm^+F5@H(MfZc=iM(cg$sK#h#Vv%ck z=30sfB)`|j$HKkFXVUlVQo%nB?p-8|m)V;So*wkD(D_FK~7P9hfX3a((cqla&x;p zMNJlc8gV$%snPLGQMg2ft`E%Wd*{Kc->g{gW3s|VZa)?*F246Rq11iejYhqk!=oez zz)8p!?w*gtu5SAxP~;5np4m^KR~Yf@4Ba1sO^%rRhN6#c-Llq`DCsy7g2)qvt~ZKb)OoaHM^^?V|}M&O|e@ZQHhOdt!TH z+qRvKZQGdG$;3`h&-1>u_q$K+^W}W(uCD6tyYH%A*R_6Y{l7U=L0RGsWoF&GhO3UR zig9-rfg*KiRDASr{tnn$vZ#^gGl<3DV2L<*H^#yu$4=+*TwUW}URv7MRgWUuU>8#P z3^{=-6DgSB6w>`2e#PU}n-rc%x4=fav>tJ~>weS#M=^Z87l8_q%}=lp|TT<1mC9Nz`13P9$m+I?R9WD8m?zY&#~YwCQ?-C$~JP1 z63<}LO$Wr9;#FS0xNbAP8b^-DPkU4E;D8o*b?M0@8|~iEh~3fpI4zQ;j~ALGdZyp% zPug>4m{B+za5iia8-HN$S`_b~UY4fbvxZk@=r-FY{nFt2n68)|=a?EvWk{zM9VmN~ z960w9Ho0ucSmdTHJuvD}e|7)H-Ay?P6Ue)buG(f4^ii%q2rQCE)}^YnW$#?0r_@y4 z4{q%8_y0A=Y0ik%TTx%zFS+9S`p=&S z4GrT=fQH^?_XHHZmDKbsTi-gz&i(z}nHz7$9Hy3QMY@nJv~PRPszQ-L;yT8AWh;h| zy1mp7{3egV8sKkx1H5T_TiP=>rP*xJw(|~-bXDyodZ=@Rt2gs~@VZ3%YtPp#Jkl4} zg)A?=F^)Zwbe-%8*cD?PBAXN--l5cq3(p|6BFQtq5Vvbz&5G^PMbfJAxO~k>z22a; zS~{C6knHQ&I(EHMATdZ>Hg3X{3u^~cowt9~=@c7mdE5|Nv^mP?dVldj9IM!J?t{Rb z0(}Sehq{*0Rh&Jtj0eRq$I7t|0~%`78?1&1Embk)8roOg+BS4MBYAbY&S;}n7a(u0 z`8H>F(`G&IfG|^FQ`?)@oGJK}FxF#xbEP~c|48Ks0fei0-4knutM3KyKG{QM&)V}5 z7^Dm%$h5K)^z)&W4xFJ+N$*5ydMt(Ov*)|25FPkK0+FQB*(aicVk(FICVnxtEc{A$ z#do(l1Sffj!@%P@OszOH&Vw|oM@JTFn(0zaQL6CWFh?7$UUHbXOp!5TNJ&)pbp%&SclBu;@ghWnU$e~@rO z?6W?6oRZX;OR|C(Q$s!PpslutXpSMb4z-0C2#ubf%tR%1CNdo{36aOZge7yNL&Y8m z;-f8=ETz3X-Hq2>au-E?0Pht*z|D`CrwZ8)S)v~%9sr=ylu%u`LybFKEgvqT4aszu zNT0s$ei@yL53rYgO#)XVPd#c(L6gs;SN1Bw=!WMs-KNtm&UyVFs)9090ME$?jpY>S z&{>i2V|I(;$7$7)(?kxJ<)D8Q8kC1tkhF()_HGGvHAeCwyduxzKYphcmcGHkuV4|9 zIYG69HenI%UJ-lnN-KqDLbmz`vkFda4A{Ol7N#PTeT~Bt7I@BM59kxaIx@bo#Bv8P zo3KAV-cQbTgjz65D{}|JjB;9ZkV%|1YV2YIu9&kD<{Hem3w^?#t4j%ZPLV7$oC@aa zNOOzoaz0*pH-)pzd(>pO^VMcU^eSrLi6Sfyn{1N))?A&tjZ(eyt7jGYIN8nIh$ogs zyG1ir%ZziD#~ zRcjg8SC}QEmhHvbEh2ork-$fGwVh+UdVG6i1^`T>!=9aW9 zSATl4e&)*wjxL>|N_K=A@@Y+NO?IK_{E{8Z+1=Hx`MC9vD@@SIA+!mWuIa=1FZT`= zG989I|5;qzwB4K=rT&1TcuX|7tO}nj%(Q;zL~RoCFfdd;O*6!B^&WY?g2T81b1Xs6 zmVC{ik^(1t@_G}Z=m}6lCiltf;i2O~Xfz5tGy-~t9ZIM!Q&q>&kkn{>f4 zlB~y2<0M#Cvf!*)Wmb}M<0YHPdT?@fcQv$Bj?P>T^%QkM9h>k==&UN^4po}9ZR+we zY9^ZKYeF7h!%Eepm?x2Vb~yRMtOZY|o@dY(Ql?4ww0ao?V^NV#;L!N2s*^`|_THZ?u!JW@uxV`kG1)iT#cZSzUk?N;~=Z$-%Ozde< zfpES6-;lzdIR17l)yJUGQs+4jZua4g!*SKdgnv2mLwQV-s14g!52fU1_;^O0+5Suo zdN;?KvVbS&8b2_}h(q!GYe@dbhhyXOQMYp44_#f|Ogwhw{*c9+ugS|%!TLX%TN=?$ zrBfkWNm6bXcR+=_wA784m$pb?kqmvum)V>&i;EhgsBlkZo)!7>pW#U(Zf;1dzIEH; z%R_4=^(*13Ak<1T`=v=MqEZ1DZ)hZ%a=wWnp6u*iCDPXbN-Ve>I( zKb`8yr{_kb5|lurpiz6Yy5Vj zb?2g(0FAtW7KRq<>GeVV(zQ1bN93Uq@?~7xA3*o<`g*$pGg!({2f|V;SPg42TrrQp zUO1>f=YW?9zmh>3mr#W8r|u%2=!!#({)8yiX|j36a#&V>uDfqb86v?{@(o0k!f7M% zFx&*>Fzst}vS}DzF#&0fKRNESz%Oki5W)_^8@o81XAScsBtG0QJ82hEXi;cE2t(;+ zQt3D=V_(j$3wClx5Cvogzv>I{#1+gZgl9>G^p!H##4JQQE_i7EjIRaMdE{ptO&x^9 zfZp`d!(E&utPPzA2skFPrcH=IeCHEhSG<|S$qQ~*Yd&yxXr*2JLI!?{XC#GWG34v` zH)3yL)~i_$ZpogIdR#znMPAjgSV0z=$&s4}KyxnDMZ~F`;;T)50!-fz;pDS+L;9+8Y7G>koGq_YdeiH$CKQxd=%>>O4`t z-f~-wWBV5dg@!C5b~ZRm#9FaIx0;A88dJ#QML{L7>r(!Nu8>V70T6Qz^ z^nR96@4?Z|11g=y|;i6W#g#dY>9wUNmJ>k-%(v<78I)_Ak9m{G$ej zy$cg0R;o)X!*7-ox!~=qR@9^LhSK#08=GMGyk0r;do0?|pp=&<87Lj4gd~+>DHg>l&A-EIAGB{a(p^1yD z)A0n^8X2~O>0SZlGSGce0rgMyAoth4>#^2$Anl()mFV*w9Db?$iD%F22 zf33iheKj0v?gmQNa9QoSJU?+0PoA;s={(aVp4d~0)v@leY&~jHbPbnI%kB4ZFOY_2 zrx2Gk!vCR6wbC9k!9?}o3S*5Lb3F95{E4aNE&r2)&N zV1w1gHGJjh15$zlukky4w4d#1wyDFiJW@f{jubzPvYRu z#^o}0McPd03Jrbg_1@JRfj>*(_xD__=BW;DZY46{*~gvV(YOVJ`alGWtGH^hr~lEG z`M3iP(rOZLJK9pL?6!}!RKJ2-7tWJPJH$hSO|mZ2=-rSVOy%=={3U887+2r)dFcsUzB1~*g;P#vyq^>xU& zB%?+;uGL_-2=;Q9bXNL(4a(c)>U^@9VR-c+$!Lnu{TcM@+B`5CuJ5mc1w8ts?6nCD zzqKk|&~CeY)wp}%9+cD2rlX$3D)guZWsm>%!{b_%m~!`S&w31_6^)&Jac2#wL&Vw2 zRC4^Q6M2=s#@wJGR&1q1AD8n=n5wEVRqqe2woLIui?hL8#iRSLN1;y$M4T`*6_X#Y z470k;_xn&dMlunysB690S`~GtOiZt37AGVrbm?$zm)Qi zR0?o)8jxap+#`eemo&c}k1E?yW^c#m+ZQ3?`qe*0cxsVLtrln~8xcs8rf+17+9mK5 z1WjL+lyX)Sr3CO8sXTE|7aa|W9m_(d?GePjw=D;mP^TqY&mbmCu`XpoT5~EbCuS$h zwbOKCW^Wzq>8Ku_y6hV(8N#}^V^z{vmnR=+Ht9Rp<)&AUHBvW*+@Yc<)Fhjy6L|DF z`=G4;n#;P)qba4!!g$#qHnsMskU^6wcCU^*J~6KXP>r=_WN0O`ndUAe)id1NK{hpx zF{0a7F`&;<@YI7g=)y7uJYMzI8i?MXj|gt4iCS^3*Skl50e89^H>|C`M{|vzDpl=f z9_YaZy8MWl93x#*a}`f*#$d^1m~9iTqujvbf^!RMPRz4V#0WWRB2O?xE0&V*#*l2L zbC^;#RUw@wkL8LgS@OrPA4%-Lmb7$y_Y_5Vx4h0MkU?uKDS%I;UYd!EmotYe+Tjz+ z@cocNtJpa>7<~2@V)0U^snK`eG=~QE*G}lkNm6)0FYWqNZXI{L~j?wb{=dO|n<%)r>K^N2pL{@XBsBV35g@I@EnMWF8 z2mUZfF~x0ty8mErNw>&CE_4!oW9|{e4D1HI(_3lJA$zAUg8sf{HI`>H#;Xse7?+=5 zua-@ZFHoz+vFFo^x%1Z)Lw_$*GDD2hY z)Nl?{RO)NGojS~Y3NFyRN=KgM5BlDO2SBM^5G=-s(fh;Qd%Fszyw>hOtCosges{ZC zY4Tok->rOUu;}@cNGCSoTcg&n6EAv~LLT>_8|B{F!aiOvXr3qW{jzZO9U-GMDl(hx z-hel|cpzr`ahXI{)x?rcmy&<8AW_|`jF~Dj&0}kfBzwYkXkNjBrlLFj{h3kwBFR@g z(OW#xb0o!`d5o(q{FsbSzCKk(Rm$3X>PE!9b#QW(i6|lGjl8K&1TCj2R{^K3<8MCcUI5o_t=MCG$fRE z0wT%w)eb)A#|*CW7_BK1%@=QGOc9fi#9_eEtt7`)Mm8t_02{H>cDoHX?7cj(0$M)+ z&30@MhL?>by&6Q*61dj)-6~FR)yjlE@Kc#_swI9H;euSMR!-Y8@de}%ROt;bdH7OT z74z8^PH6qs8*`m2W%IgfrEYHqmU}zen(bGx_XD*Q^tWcH-33>WNEAHbnUDcbL?Nz7 zZir3gl4)7pHK1esT$<^^Dt-j+aS_vF)+Mq%3YaptK|P}lpH^|)mAm?(N-NxZnrsba z(TvIDX)ox=m4b;{BAXueQs{Fw-Ie6D@G$YgjC19e#Jl-|kP1JHwoFEgHMu?-S|-M} z@s_P>Jaz0IdnhWriwVQ@4W?a)Z9>bncV~0*xXB?-hyltz zS%cbHqw+2ov!mg;v;KU|=_PCT5l=^Hn}RY(LM>_)PHQ8!^qZo~KO{*Oy*R;zkfywOQeso_(G-@KVRA6IfU ze7fz+LR+RI-<5$_vTn!c&*zuZa8bM_4av?btUWbq6m;Q8B&XaAFfykR(($ZjSu`Qt z?w={ciL(+Hc>AWXWS*m&TgK01?B^HunZiZ}L7g{g0|p zZamDx(tzzz4U@OI47Su6q>L=%N0xia}lVN(Z3z3+rd`l+X& zVX|F2QrCA|J@tmSCRUsdDpN^f`X~sByQqb>YH6sT!ldCesN2a#w!4KQead`r0|Irv z$C&k1(5olW*0d=$527ghDvaBvD?RdKB@sGuOLll+l?W)`m34Ko@^VNs3EdB0a1ng0 zzQC`4E?P5Z(TGJDZscRL4WPRwb}|Lwyc}bP52rvrmd$LgSJ|zUJ)OWER!uh4{;+fS z+0Ox!weZ*4w#^1T_829i&kR$rMvnO2R*fgfb<-EpkYRLAY9Gz5PZo*}D8eL`*abOv zH$WAwPWbXr$9)!Cc`zJ%>?{ql)0N&Y9!~xuHF3C{cC$%nbSRe-;VKUEHN;&??x8On zdbj55iF^BZz9=Q<%D7XX3UZ1#kIjuca$l}Ylxk4h7fo5F&*H1&hV&baakoj3CM_s8 zsE2v?8LKe8_u@dVDfh?*?mKUOx@-`qbFrU$`|Li}&n`$Fb$f%hzXuSH*xPBV7QZ^g z77|o3+$tiG>4FI2t^JBw5mERFbbg|5XYF}8r^JSzkJ<5XjtcfRy4e2vNex6PvEkND zd)u?jHVb=J5`_7igGD?%voq>hzF{ds!eq4|TeQwyoipX3JwP)xPm;NX)oE*x{kF60fNxDiw)+wEoyvsX6~P}R;J4w zC>P}k=#3Fl+HRbmo$su4-KQ}QKHE#`DF|u1`KQ9gS=OQP%v0FETUUtWvO|7=(K+bp zk^Je4)b2s`ASYCf!62dabI0|^EFuuMiPTR&YBdVk48SFSzws~5L56wxN`w66`&NK8 za+A=k(xB1)NeWLg)xvlR(GNmWS%L7f4I20@AXQoa{T$ zf`U(5x5LZZ+1b9;26NKuo`bM*Ko<_1?h(|<5s{ennT;)?W2K>hIrK(W@1pQwzb_AU z(T|mSqZi9o8g0mKppcZjHvupY=UcLqH|EvCLmC^r&sf*jbFt*OtWIwHO*hX3Uv|Q~CN}8)ST3|+2$c)Ar-mhA-N65t!i4pte zu7JwnBc2#5(`1fHaC4xIh!3r_{z&gMZ>x8yV3iLc*C1lAg2I)D1{sIpF#}_ClgGwJ zVs9E}}Nam;j#%K*C{OPg5#lZ4@26SML zzKbVh#>!+u8aCu=Xgz1lPeyj$H;zRzYg z4d>1}9((#oVnfrp2<86>e7ZKjTN&n+4i$aWCWEARM!Kf)#%Ljc%bJjj$G?O|y-8(x zQ2+pu5e2~YP;JsO>-qanU{}MSJ+3$)s>HQ|zTyVE?)MLC4lcb-(g2^p03H|X42OwAu4b7Xc;03ig>cBP6!53QZvNw4 zYIj`P77sTB(u3R_ybv$VZpya(c>+e>sncECF&`XS)I}E+r-{1g+gLX!OG!!vavO2OZ8mXucQO#(fk+-$b(Sl=JgetZfY*oA8W9hE^99Ep{ zRQWtBN1znyWr5#BL)2F{ic;m1$ne33B*2W9als+?6v0sRIzkgA?s)V%seP-&?sts2^sF{@XVC_S*G%ax!~!n!eqGjgtApm9Tr_ zyy_owZmGg)lT2FlRYT12E~?U9*3FIQoils7cmXyUkib6g>;`QaUdvgsd`h=)_X&&R z;VU5ol^h>iBYU#EXTb8s$lbm6`<7nKtfnVRmg>~8jX@brN7js;4k3(f+!|T+>hHf843J5aZ@46spd4Vk)4vnYsWP*nE9aJdiU9_!C z<`*%M>#IivwT=GVGofk-zu;_GxUgZ#rL4zp9M2zcJtvH3SB>_4I*BEuiCxV;&aT2= zn*6zyxQA^-8ZATrsG6AhxuR;EfTCph*Nz+SMnBx}(S;(#El#)6$;$5^DBjhYMX-`y z_=c@2`D-nsCg`}i{vuGRP7hW00b(?wVy?_Z8z3BtnT-)G?hLUG- z@8n_VNP9j(s$!qMNpkd+gGy!-?RF7o12BMlvR&~^b1(7k^hJRm9d9VDh{}cOrFGDC z=_Ow6Vh8Ki!(PEy{WpSt+Tp-QHR$TW<~S2xt}=eRJizrv$y3h4=y5( zY7y4GeKk2%r~!4EC+~7Tyu+W0TxSQ7TT~S!BWk91dKwa^YNp3G@`+1!6K9WaZfQQM zjJnXL=0wx6gjVSiFSG+zgi1JjIpyrborvITg9=XZ260+a-U?l7?eV^s%%qcw zspdRXgqlWleiGHyGMAonRH+V9pPmtv`+`XoTiFyBbA& z;j+?11*)H@rTs24zOEUe`BECr@h}bm70q12LJ0#_&9!&;Y})><3)d^9yQ^7EO}M4k z!CAx-aocX5Idc;nl02-!y@2uD{+P*_|G~RkiuYjmC*3hw7PORE+q1|K&Y*m@^wwGe z-T_htW`Xv=a&|&9#oV~rt(I`~GS9Q)&Bwb$aulzxkBjH4bzN{3)B3&d^r7B@T)uGlk7sS;8C6K~ipgYDW|K-kO=J_iKYp^$7D!xSMtevt`N9vpou`GEC~XPR9?&=nk_rMYoh0p}W6k?{H^hl} z(aC>U<;EL0x-xQ6KOgS)PqxM6YQ`1ymYxkF7gBA7h4(BiqmpczlAZFK-txfPjtK~M zE|S$@Gy5=48x^$zkn;54S}zI9Az=95>s?e~{mKGQQYXhO)Qub&yE90Z;+^ODWQyDA z$l@TB*c~(Mb4F@uA%Fb_L4OE(fk~7yVu*XLLtrYk@lSq(cYH?QAs2cqZ{eYV`Sv5y z!`@JMSQ%1B1KRtGm1nrXGvxJMF_&}PMm1+o`Q7>)_*eH@tU_$h*m023AK>LuBf1w% zB3@>`TDvHpmJ-~`!mb`EYD+F0-3<+iL~>QGmvTPVLxq5`{I&zhPG!(#C*_i-%!N+( z#}p!00^K&yWtJ!`MN}&y6y=ru%r*x9nVn-_d%BDjh`^ZU} zy2Tod@y)rP9-8rp6z7vl!@F1S#a_bphO;n7A_t7V5RLxk4vxm~o4Mzyf6oUQfRPor z9B##WsY?jK@$~xi5@&SCDGatnCbQiaF+c#LlNjA{Nh^5`C3X~!cb67te66L!L4cg) z*0nXl8_*2}iA+civ=^pGniC$0x)~vjhcx9tX7fj>eFpNg=K16gHrOwq;)FMy_~G>Y z0;k}K0^6`msjaqu&MfkAl*#KDwW$#H$Zy{839kk=4TJhd>b==qa?L{O5SM#UFP9DjI}XI1TnUZJA|od#f|#}YmjE$;l^kr>KDQ~;8s?mK_^02|ClFgp_6Guc0dcK$t91T z!wHuqfP@&57;;4rVd5pN*BD9TI%@%0WWeGuoc|z@tv*vgj2NWBJ4RPztXi`?Hjq8m z5WsAyzHpPp7Sp<`CFeD?$E#~`Eg(iAZ8jRN@4E2M`jX*@{zhJl&3043{?&%=SMh8M zIN7Zsq9?vMFe)RgYqUFzDcqIwl$Hev%~?F%$7z#qv4Igk^=LHgq7*w!4mE>TGI;JS z(puH~6n>s^5RB_ivCreqOG-&L#KCW$3VQBI?s&sJ#gnH{J^r+0Xq^BQOf^E}e!kE69IvRAUPy0cXQgnHC(c5xqzgre++;ZHsH zELxb={7?zqgN@6EeNt(s>Cz7H#jcA3m&l}?n*dzjvQ7_(<ve zy#Y?h2+%g-tTLyLLf)MZrtwI7-;vuij$1$=tHBMwo<@OBPkOOA8tIvfpT%X7$0QKy zrM5I9_@zDiEipFZ4BPYU3a-*3e~lfnPZXx*A2=F$PYhim=NX$CZs3clcLZ6r;0GDg z+h1nGX|3AD+S5 z2D++n(qF-11Ne#95(?4Txi@RZl`slS`8x148Q#AI&%}V|ymfva7$7AIVPpRCK@ttu z>jE&#sm@y~^2{mAACl`NZ<49HHgbN?jaq>CH}9Ds_v)NtC9r^sIgLZrYz?c4`<&5n zi4(IfFBeoZ|Nf%acMoJdQT{i^)BkZ^^Y&Qb0I_DNuGG#;1Wtb~$NAP4gD9V5`A4ME z&4{IM=ev%5{ebZ5tgopGaRZ9t6E~xC@x@=emU{TyRP^hF!_PS8TM4YU-^fjDZ4H_Q zK%fj&#;j3d7y@@pO{^V(V9i14RCvQXZ~I>>S8iLU{~Cjx zti*fS{>@{a%$k2^+&e?x%U}4U&BSSN?UugZfT?-ip25;4nRw-G z0l;XwkY1YE#UGCr&oFX6uA$%YNV8>Zlq)usNemWl5Gx(^ z9m@M%5>1e=Rm9{mgw08nTSd8ZT~gciuDM$eZ!rIK@4{U-I(ekIIcTlSSGl4kNz$4G_6B zW96kBSdBO95*Q|57F?ipFMb@wOd)||$91=Lg^0XaMW7&^W=u#{xYXhw-`B81Vf01d z;5WaKzEWT~v-$ILVt*CrQ(6sI2xLg`^tR+C$6Om;9LfCgih^2N%4FTF6ZBQ7HLTL8 zgRVhSWy`dU8xkH5--umeqmPx}As7?yyyeML%rjAS0{9J;{mT?a3Rsf3t*Jl4hzJu& z)~Q)@{ba=JsidI^oF}_qlKh{xFlluzcL7~+?QYBQuN$v_T_zcF;ALvFuSoSzBAGsj z_S4{PbqoGlscM0ltfTEE%#xh(EAnRYmucdW3)3cjjDvg(SCQ;7aIrtI}W z;p)Vrw*NpA)sSsVxZ6FIBR!|Rl{a0wPS)vQloc%gS6jGhrd-j^ahk;lnT0A+i(h*Y z_m_|jKAs+muDfyj08H zBh53QW$`4jyi)l7YtLs(Eh=AAE=?5+tiIU@YRMnKkf6; zT5SI^hW${v$!2Y`1C>?10*%RL&4Rt;C2XneQ;lx5PBNaWVn9c_zd+F3&VA}HxwjdB zNjvTGmoAi{LTTlG{RHej0w_$v~jQh#k3c%<3l_F;m{X1F*CCNi4dav zxw)C{B{I{EX9PJvSgn?)pR(w7`2F?EZRSHg7$s_O_|@GmKw$X%b#VO7kq^91vd^ti zKn&6#Fj}G7)mf2fjY~wRb1^;o*-@;Dy|!+b->xE7qGnp@EphO^Z>~e2{0O-F=0CHw z0;jIboT)7p);uoPPALtgY9Y7vbEcux+ZSE>uvQ5AQI?M zKkSR8DXPfk(jafuAa>LKzn$XwJYmm8E&A$Sg*nxqDHTq}o~X2d*CnvCBKSu*K=t5{sb9!2Dw zHggXuua3|+q3sjTO0#mmm#_ACAt6&Loyk+T^BQ`vgjk~qx!;V#i*QA<6!rh#Q;n=K zwRCc)~vVnV9!FSStYxv*@=P8%Y+0BXBqo(>pk=nKMCCQJS5mvEvi9ywz@n9x#a%YB3`%o3F)VV8$~+>HoEc2Xe&1-;&$kpx z)&BAEh+FUw%cojR-QJtFStRLx^t5)tESPzGz`jElC%1m69c;w;+FH9YsNlKX^GmSo zcw=I-{h&}Mh2H5uvyXyqm$A+6GCagrg9-H9L{v7p0w z71bj|&F)IDHbG|jm!jx~qq>+Id>EYHs|FzY*^L1$^(mG9miVc!wuv4+VLu^qa~ONQ zzVl;yy=bvV&0kIT&op5FUHlf=<|bd~U!SQbl(51V$gV#(ne95=H=8(fU9u zx-Kq&U`hyc0AE58s)rbuh~&WiiL)&G4V#m_2o#bw0LrxqK63~+f7lSGaV6y5-2XN< z==yhep?4wjVQJK#Bmu6T&jVy@?MBGH-9X)$lbn)^>;Stf<{0yQJUpK-1Lyuw4K-qh8Igowo(+a~PJ zwO*DON=(6c>9DZ7-79OKhJ4Rtq?|~*VHvy3X*4UK+q|_U4U5?jQ?r#V<*1NTXtHdJ zYt6T|*5ypaWWv5mr4>uA7E7*Vo=67Kaq~-5Lr`wDK1ZZnNOohBj&Hj7M@)A!QN@+@ z2_IR}sI}qkGWZ1%2<56lfn;gqTk&1)^e6|WN`CLUULqS$!I}udv&r*AjeeRei{bvE zE!3=aa=_a*rA3r4Q8#Th4KJ$NT7hN&>J_FcVn8E!8OYT#{F2?Y*j+0x;g#H`>wfg< zo{~dYvdmO?St>V*DBui-^q}?p*;Y~ZxXq&PUjx_AaWuezHU)EpVCgu;h8r}v=$u5A!WWmHAH)c=Ym%0NW}rG>GLjP!N^wVCl|^-|ga{FBCV#wEhQ7^I$V2BSEzw z$IoT>>hciBkd%-%o2cD@oGEV) zbW^_seWyqYNRiK)pGMzHTKNjcNc)ymu2QH>{~U71U)X;H7U!qJOYVQqk|FYi=>b^H1CG3MQ%4XV`|F zRn+BublThHgZ?Ut>Pc#*!iferFw9g*xCdwH;|U8sE^-B4pDr ztEJ-*O>r1~omt;fZ_*u{M+jMocLq>75=w;*nMlA9i!JSM#!|i?fC0B)BFAl!l<-`x|m$R z_YzLJ?0b!2#iUW;i&Vp-3yp9zsB7cE=`egzoeqR`Tx3z6Dm91?O=?IFcircoltk~5 zrwI9AXSvVM#Y0q;bcSk1U4UgZ#FAFV2y9)C65u*dIMqT{o`>BEg&rzfcimB41o&X^ zQYV}q5OjCw{`4mI(@@rKsM})to*O0E3iZu_mCd#2BC(s1*&@Zd_Qt zT{RD+INfD#URnjX#a`43g=tWw3#8oMqk}ypUu;VzRPR&H1YM{Irq+&9#YpbjzzHg;x z-Fzo0y4TXTP|nFwOQmxI)lN-A*b+$bd`4fv6rmk(eqz)}kj6}7O6%EPg^NxAyfHbTcox{QKMEGy)cK~k{G>O@Q%XgUQ6~vk;}wD=H6bZaZD(XwuC z6rC9FLXWD0TLE%k-({RJNuV^cCXWHUra#$cyyPt7vzRR`a{0Twkfu6f9HHdPDILuE zTzl|9!Ep-G&``#4#vhUxvOI!8DL;%^s-OzsnR!@`0-m}_%+*3y9c3S3vYX!3&CH|( z9|6hIMd)5Z*{$N(ZxA}IqNH<<>Fm(&3Dy%NgY2Wfhh`n4Yr3=HaD6xV<1$H0U&iK1 z^9A5~x=y=Ull6Gi>v?Ve(&b3zY}Y${{5j=I_3~pgOx=({=)Pr-E%%5|O>4TX zIr#3NV#6u-)3M{NBRV-k_w1w@o_jA&>$N+l`K=3+X)3$qKgwGSlh8dqtM09j%|QDI zpuI4`Q=cY$yiuxv$IZTOW%&THGxODIc%$_w!JCwCsGScjK&l{cq%Ie}zbDGzZ>dKM z&(oU6Rh{Rh2t~8awmttoAzbd67|384tOM7hK{`Zq4%PieMRUR4TNgZxKSEl(jqJ3^ z_$omTM?=GgKPD*V6R2NZhNXEca3`v)Nz1scA2|o&i45VN5e!nC4YkV-=a*tumhx1c zv8#S?V5cS}|8mfunMq2Rsh3BR513p(CQz`rYm?;`&Gw;4p!_#|kbSYnVE2Pj(l>tc z_%m0l{{h_iRRipep3rmsw)GgP2ZTbt z9k6fV5wja*K0OygZcCQwEvq@C@=a}=c^A;h|Lyc_4tQ!>diDM3?#tWm-u-yQBOdMO zcwwu@g#FxrMVm0eQgc=7I)!XE{uUA`PdIyKL z|J$0SHlMR|7Ej-6Ek6)7Xl>@%1e2q0f}pc%9$z{)_-qR@-Wk2)vR(mfP7Gr{;eMB4 zMrZouf378w-R+wR#z(*rW{?aGy|V`+r28|^4(ou|Ya3#7JJYA&n?&aS^FwVsja<6i zrKzZNLlC<^l-n(PPd#-3yI7H>=t{)_8d%H`G@%10_2FjKI&of1u?KYAgN|~ z(0>1wf}x!-y(1w=;mmzJi&-W&2OMe*nZu6PaqP0Aljrl#$=-cP#CXguI0i)YH?B1J z_{h0?`e(z|3s4M#VJYj(kP%;`3)Pk#v4aOHN`Ou3PF~`AK?5;fpu;$jCK@mX;Xnwp zl-nV?*~sUX{Fyw^Bj-(6OE4yYdc+E?xO{($oe*?e;PC1kIa+Wu1Z*9!1-4Ew$DNs$ z$smjN!F@{ty$xi0#T7^U2Po*gjGqy{p;HkbZ6q!4Z{{C%@Ma4TPHxpzM}0_ap;53N z6hM6uUf=)*zO68u2UnIXU`W)4PWX9{X%#j+I*PPT;fEZ0RBsZ#$S);%E~5ir(YWMf##^{zTm(aooEA13Dxkj7I`8 ztB8#eB(TcKDShLzl&ODs2;uGDbWJVCF+m$k&@9$)Onv!4EPM0*=a9v zE|#Y*$SoWMhrK<)9eC^j1;8{2TK%HB6{%rExT(E}8#m8{pe|VKwf>X{9a1sgVVi z4e(076%6>2htWAwm<#XnrFWK*U;kYJSNpU2Z^QMX3gD{tG!Qm%r(is7Af960$UOuw z{{T-204ccT{`o%3Zb>mYMn)eJy%a2 z7Wdva&6Byy5V)>i9b-;GiDp_~TeWY$)pAOYIZg<3E>Gm;>J)Jo5E?s8zB$CrAG972 zm02Aot^QQXmSeqXL&nt*$taIx%z(E#OkM|7&eq_qZ#UH)p2Gf=?c1t0czC2L_5~iE2hs@Y%>a(6vH`YMoZ(Nh+S(aS=&$jXI`+$zoL7LqR z!&p=8U?unb-+(oWsH?=kw~bq$s@W!_GzA*~rj(w+#Br}hga1Fb_jJRv%j2>F0x|J$*pHEj53N_N`@8+YCc;>epZ>n56iuNY?^l(WZiKpVUquzaa%HH!?? zUF@;KD(W_fjT7xm5gpxa+2-MI$(O(B1liDv6v)lZ8R=XQ0M}_R(jDkmH`;0vsmoaJ zmpp%y)|99Gbkmo(XnR|Zcq)J_yO#&4e7-K`qi}@dN6(5DDQ^v(?ivkRuR?#XdevBv zGC=E5dR=Hb+7Qi82usAiJl5D4I+zgrPj<$_NytnNeyE!mU{u#HcBdD!^m+oAC6?_4 zQ;;`mlQe`rN)Oq4#*_I^aE}Z!2=A&jgbEtDM^C`w-+_Cd-0htFvyJk6UE(>3K%_Ei*_WO{#Wz_ ziA5y)_MhyBU)@Nlcu!5~~$NtV% z6`^YM8&TsoTzm;Rs{t+Hxt{@r5Hm4vj(hPGS8b1Rvhzo8d|(Z}EQ_4H^3_a~>N`nv zyBk<+;V(MEgLG%k=}Y{0IskMeZ+2fq=Aj!(CXB zq&a)kew(N0+c>mFO`1qi&N(gQX=zvIlkaSBfI`rHJX?5TD^t{SG#cYz4l_}H_KX(p zoc)$M)%E`H+VKAuTt27o%b_v5GTr|32H5`eR{TK4c=Wi>snL3!A#x9lz@zu~YiHW3 z`3Yr6W^auRhB5m<+SpB3&>40}M38ooe=RfhHa>F{ih?u*@z zb3#Y2)h(H(Z!Ujce~3gj8@}}8iqSeZtwMr(5u(|Q|3rrCv>_#=R$*%D6I*I2=tE{@ zuY;m`PK;zO_3$~);V47*i(!+J4f?@y$;O(~fK6xWn$pwqX}9GU!_yo6tINt<-Hx1W zN^Bu-cZ0QsP1Ukos}`&-7F5$!nU;n2ke)&!>eUiWGel3g&QBtnP`YzIv<3C0^Fp8e z*o;kYwW(ssG|$omxTd4E-mWO;ZfqBg5`B%4f|l;#QZ3J zj#kUrM=I&36bxa#PW^|w!DWiy22P|x{e(eYa>?a<)wbdmvZ`%G0l>pzc|Y0Wq!i!| zEj3>L(#9Mjuh@{`G`&&O#ad`#fq`xc7Q)2KMx&jcG(Jngd1c8R_( za86G9F4OmVlZUsQofkRDziIPyq-E2q*=RajjuyrEMLGTu*L$^uP+bW<@!{Zs9QN;$n6^53EB!atMP;^XJ~&%)^}N2=IFPLD<1AmDa$^h`3)u> z7*F@qCVt)Qc*0g?M+*h%`uxq&H*!|*Xnh9QI=Acd$UKA1r>m^aAGCg1h1(_?DE(52 z4{7@Wvi&Rpei#w=%kzae!~7*{4rS`c$g#74ybUiQ9;}NcolrjgttUz0DPawy(2fhA zl=z~q8x-(_p$~(|aiG0J(-pkchTqE^X9UiWyV}NiY=<{&@mpp6RsQ?g_ph0wHWE<^ z_OR;EYI0@zT5C5mQHnRqkzb+fjuhW{Wi;cF0nRWV#t8|pA8B4In}QrQ*9~u zE>duEiad!6F!{{1z{4qWAuPY-Ga-h1>u^>Wl|$ll!U9oIT(`aUlFxz!9GfCHE!CKO zR@&gzy6j6%MP|#cT0^`NOMywmg+EiZOVp|)nH(KDoLM&*v(AT$m(MSAxVTNp zRYi4QMPi$-qOY+I+n7Re{0Bg?s8Eht%Rw zeE25CwMEsd{N;skOdYIk6;a56S5lAuN!G6FFmN@&B`L!sb+!>(3vOjh#dg>uRiPnK ziYt&BGB8G+ZG6cRUKx{L7M4gg(7~zj6~s7rFhiXVo2ar^7;$fd4N~ zU~#$_h}~k;?Jzg0Z$(f&EO;X#*qUzNgWKxq zf(H|$xN>_bF`ows*cVkdU06sg@YlsKE{a^V)D!dBXoF?x;v}`!6}Nay?Jz5fTuBrP z^I4LCO;J6&+;U;T(dEOSD6Zb+CFpy0#d=`{Yq$lbq|@bWugHiz?1=KkwL~$o=n7i~ zMx@KhXOtJWoOKo~NQaeKdB@_Fne_`@R#uYwT-c88{*u!AEifFa&(AAF7Q6s1tVRd# zpvz5g3rvP07vj2OnLIrv>_wM-*jlOFvMbwREZAJ_Rl7)3xkyyE$XB&As#)YyDYa%n zO2bR^%5>!QM%0na*a8pHF<-G?sv!^NK|?a3EQ~|{%t2Y022FW_Vi<-#X@CNl1s&Ob zU0@VCXZdMN&r^a!(3^0$P&P zbixJn&spR3^EZ`|rrfVzV@6kpJ=2z-)i<9?Z&DJk)0l=dqaJ5kltfgJB~RE*47H^S z4cXhiUvd$No}(k(BffVIPXDBt?*2?$N{q0V zyOyk%mR{{W^IAB)*mdQtm`L?#_EhhmuwgF2g3%kjcZP%pqgk8@(J^;HSK2{ex!s;q z!0kn@BVRz&;<0}|fUuXiwkv;ruW)^L`6|7@wcOe>dwpxWjpwy*)^W)$=+)i(4qe=f zTjNE!RIhDKSK-}1hTI)DuUEF_yK6JOuytN#i+f#bxTF^Lves}7-Q9!icE`-D*0y`X#F%=R+`_?NH!uT0x}Hfy>XmF>yg=MGf!&1Fk( z<;sm@^-fT_mzQ$yqLFf zoxGSGsvoYAmqPDUev*z#r|AyXl5!fGhWs&pFH>Xd|DN^J zEGvI1%%0)0V%1n-buYGBNY^RomIE!ljE$Z3{a(ddkE(uuh1+wuTd(2A@2UuT6>}$c zcUOKs=b*JGGXMG7?EkhN$?FnA^iA9m`E=WhnDMZ!O^92)6zZKg$j!1Sjfklo#Q?^E zqvDs1BL}&|dkLzJkk|ogK=d%)iZo;ogLnhNhsA(KNS_9IF2qk`^(|ZU96WY=Md@Pn*$8fd^BmGn8s9j z;F!i#WgwX*1rD%G^8wY)tdN$zLnn!t(w_sxgT_Nl`TyIyzUH=ZB>%pjLX}EgW*;Dv z1poScu5uD*;;Y1V$@Wa`lZvFsShupImFSr``SJ&l5+#xV2vP(|#dw(1SZo3f5NLFx zyMNupRMaZ4RH;*Pg_QzT)ru>H2C7$JVKh;rB1@sQy4DlhYHHM4+5p4G2`28Wg0)1e z^irU!Qu&3<9fwuLWldGEqL|i`0IP{@EpZtsQ1f*gIzE8D0Rap`?TCyN6y&Vd!Ijiw#`OH9!CrFVvuBL0=PT zrVxs!P~Vi6NsfX3Stf-9Nf1|0(>d7HVl8r$J{QUbIZm5phh(s+1aXDF0oa(E^m8ps z)cuwG?da<3#pqwZ^0hp*_$cwsrfg8PEkm=}=1I2WHvH@2d$gLmC%6AQIzfAU?5$j^ zd#uNunUKx)$Ah@vOi$IUtIFUm%`sYDwyK$a%1wh}Mt*~%WGVh^46^2!AnEyWnvg$@ z)?u{Fr3f|GabCG^w)OII2+j1vqk5Av2b<%7q*#-2Qkv_NTHc|V=1a;+HPZUJaKwPS7q1ER`PavZTT7BnoqC>z1c`x^4r* z_bmX8;Fz4+nl@B41v-jmTj(Fv>^$;Tw*uSrO&u7X695=O7kGhX1Itqb#R+Wa`-XIB z6P`+!_Lj1*F719flBE-uIywD!;ucV9i)d(X*~Rqs0dUh(&y4y zE1k6^JV()18mPFDcnc+t2*Xr8Z)h)LvuY#mtYm;?w5DUWfiz) z+m

5^l2(uPY;fQL-_s`{RvnZR;$KyJ3^v*Li}#C z@LeB3OY@G$+oqdvsOT!x9cV!aZKG9hn?#@?&&n%wib({TL%2C!1e$zVn<9sM6hb*E z;>S~2B9Tg*9@3odS3ROnXO>Fd2TJ6ijs>JRIb zDozz$Y?0u6bKt}cO@+K)===SG#C_V49vkkHrG=X31_r<=NC3LN@+%wdz(n?|0xeTR zujo*zfVu6A?5!CHa1rCCFYa`3c_Ubv?(|AKfvu@jMpy?r{dcJ6XEO}bkH9G>v<^8vBk`X<3WuHVSFZ*@csoZ3V#QXVi4f5f(VEJ&u zT5nW(>1QYKjh7YkOYaCo$53E^I=&E!Jf%?1jxD|jS!Ba0J%MriSoo@Pz4c;-hF%`Q zk)ssN*_a8$zax-$YJ%_7sJhJCpZL)jJrWljAmd6(bdR?`cTYrp2D+}nsPf!gphSTm zE?pyuxIsi;H#DTcm(QPffiJ&cj0@?5ZGRUShQv}4NKr^9U%#Go8->(*#2o4PfBW;{ z;p$_)N_ca9mK^tMczSXY?;0=8Q{1nbE|4X{Ighc^B(_C+i(=2zM7t+)r}Oo1@AeBv zy!+=Vl3Tx#=~n*Z^sfMs%oh>Tuw@MYYtb;{g?lsaTM&Ovu92roTx>>-Xp-kSve)qv zIUa2|4bCVof~v}Jylx|W^yhw0{0MI34zA|k@QuOXzuc$ge-AF>KS*@QmldF+j@I{<;WK9 zW4G93$84qN-3gr-`zD^(=U;9v zK91jCe7U&(aJlLJ`qS0#4uv{H|3@6~eR!!E^&BOie!~OX7W^^xQACdTIWnXHE z^+omxKBXIyeaH%2Ch8$c_SM>Mjmti|!{F@uxnT|!qJQG-qd|3oAeVAP3PpQVDxZ(eRcgi@#f$R?XCIxX&T|n z%SqiKsj~A+8##W30X=(C$)F=qpL7vW{rJnCd02D!P&O+()W`IVL)d$j&0F zlc#iVGoHKh9QZ%$nlPqkUBELh-nqt>ucLkV(DqvF zbY$Fc{QN^JmH_K$Y^9fI)yB1LuG2BC0v{(Q$3>~jsv>g@tx^%DU^R7T(;u;RI2^^S z+y&8}F*Bc#9FBu+N^3Gvc}arRvm@+y1=X!4(fHgSUX+f?#$NdsX0bYI17bfyoe~@T zyP8@2uA>pJg8w`tb2=`9@Um`fsQG3)9V5Ah#^Hmc6jbYIl)MG(`Ny}{Vh==Pl}a5| zPFiZgwT?#1T{)w|LM;>~>u7A*0`!LuEN!pTQN{WLSi@u)Jq3}Q_(~fZB(3>wm3h8G zO$Uv>uZ2&k3@0`TXF*hID`Gz!_BZ2jx%c=gc5t`L1f#yiy--O_cPmOO7wKx=j;UEp zmfCqD(CI~HsV}x90~fIq3PkR==Cnz2!} z(GDMDwiIngW2#sjt-=?il#}gfoYm^H)v^ipp(#2VWwi=%b#A-|G`CfYqeQd44bp4w z+{iX6VtgeT8M#c1+tJn6i_yPIvoUI9VbssQD9yTP&Lz{jcZ6NtvL%;B3*LqHF8jN> z=x+aRx0}1wa@^I9?nA598gA!0v+b+JmM-^NzGvGU6urG7Yhz0saGllNl}s6EPumwL zUEU(Ju}G@e{K0%R#UU{uS}bE2CjZfmdThV*R3LBH+J0VxztQOlwEkGjZJMlY8jf9< zZwG8xcJgxVq~!yWm77UYuAH1qPD-XGBPW5fT)D{mxoO0J){=DDTA9YB*fq>7U7i?Q zQdTZ+)$Z+NC~FbxOpY&MAh z>hJp7M>CC(>|FmjjUIn{6QRTXO$lO*RI(aYvZ^4y@FTql)VCPuC~SSJ8cSTrEgXm~ z-ROYah%7~Ykcn-f!35hvF;5laR8`u*wy-SUGD33?+d>O%8rD|yZ?ti@c`#5cdXFsRJ^aCU%seh29P4FEk zJMsn(7E+_72Mp<@1`Zl>vj-hGX_JJ5Cw+D}$jGiK%2}2Y2-p*vXG~nvg+RqnbI_ht zDvKGnQVv3rk`hn4^eO$s+cm(H(tI*JR5a7IG!u9M8YtAXiIm|-Bg0!>uTp})Rta7O zB5&7J2NFE+eA9!T3T#z}NFzta@DWRJD$LaWkPlPjgh_C&q8T>L#7Svh&TWSTN0kzk z6~-tjv>>p^LLQEa33puq$(mQ~rT|W*k9jJx9CpHVySOyHd~q2n$#6=o6>Zt}*$WJZmPDKb%L1n`7NvuUy3Xq+GI4R_>7Fj7mYzpEf zljK_DBMXTshzC*@Y=)H%AS@YBmBPM6S%z-yIXN6!k{d!&(B!bvh>NS13Hc~&5fjpm zi>#3$8bwu-C#D=1nW8`{3gRlCV_f8?0D&lU%aQVsNs^EXSt#o3<+GLvAFco~D2SJA zYIBj193-G1E|~euMW)ygexeR{zMSSmWO-|ld)1)z1WhP4PrLZR=}>s0wkS1IyT1NX z6S}SxqE7UlEGCaLb|e8(P7r^Y^zkA`SqL~mJTOzli#)L*+XV5LNfIYqac3abM9mX9 zDV#7v6i77jdct1*I8#U1hcFYw6*<$KWnpI^#{_Xj$Vo4Mik`P2H}rp}^{Q$6eK=Uh`aqMd~$OrM^d9=Ihq9sMlYe@(uNRV%gZ`$QJecW!llz zAiC-^!CPsdUhCD-{_FL_jk8~`d7_!;kR5&2O`Hp|!TWE zalJn1+I@rjp|j8$^OeQcRDh;R+bV&mO6l4Qs9``??dGRX!6{5=s~yW!G^-PGPUqPq z(L~>Kq}6bdmbOkwuHm30HvNg3SVAYrHiQ#UFeuB1ZJ>kya{h4r{^sHOBmSbK$K(cX zo+EGZJfGYyW>Z9TcR#wk_&Wac^X2vU;*X1~FBk8>T<+S-QD>WsSLNsDr$r1e^N~_S z2uab^6bEAvv4a>OH&_OzngI)KFp-rF4lY|HWCB4 zHuNn!Pn!=Qp|^S3np&xrHa$~&U;Wx>U+MJ_`DwldYOeQmUXoKn`@+dmX zv)Cljq^H*IZ`zrM#Opde;^IHZlqpPy4g)Kkc))g58>c}yV50jFc#iH^z5}h$hb0+G zhe373E$ru$T+JEP&C@p`cl9lZVvHF@)hC{@hcV+hW&mu})jb<+GS5V^CnaaKHhP}V zarN_L`m(=K-m6qdJ$(%+l>4cAn-_Dx5+oCCJyQ?ISOdq(aemvy4<3+aUO*>`NByvx zoa(#Mof)8@-)X{Z4{@;`B%wQVIyKA18^Ss9MwnT$Nax+@^i`n_-`}^C?d}e;n zzL!%U6{h}(bs~yvW`ieb33!sKohr(yVzvODWGgW6JV)CHPjXBZ=-$M2HOqybo6jAL zj}*w=*w||w^HVk z?}NPZ1a`hj;!4-`jnE8z(*$AYc^EWTkqPWFf&J(pNE+M`0d%d<&}=Y)=um^YMl@B9 zC4rq!f?KAdO>A_8YiP8{){_$acqDks>s3nd*DArqLXb4LBWzbwZ4cSjf$1s$Th_n4 z1fTe!VVI5%G(|7<#d%%Gf(!s4sB?Pdz6)C~ljYD5>_ zetfW(wN`nc#5Q_0%DY5mhy&;<4=?+2uf|neVoT0Avyzp%EOBvmFxHEh$-xLQIf$`3 zs*0QZ`@J3AcEMfOIVu}EuPm0=>n8iM30ms$uLcwCnVq( zGPs%HJxgqLAX#%4?FGnYf26#~6GSb*qUb=~?^qk}iGs`*g5R~C3r&vfppKQ+uJp*x zHN4JMt`2mdKBy?eIDzBD{dp|e=E62T%-<>O&X^C|G^s7X1~;|Zgor=7yK9MUnoyF{EhV-o1DHc%n@Vg` ziEYZ79EojOhbM_`%H~93oAUA^u}yh-l-Q;XPpHH;mDr{S@F%fN`FNDrrgXj}wkbbX z65CW_o9@23#5T=0EwN4O_aw1RCAKNMd5LW*u}z!kr+?U{!F)BvE*lUnmdG@i0sqmB zB*pYjBz9G1p;1c51rX3NMF8JTjdktr*8{w)Br zVE!$7q&to?M6V`3B^fpn!Hf&e42_J@kTDuG4*M9Rk$|EwH`_jm^j#m5?^^nYdm4k( z2e0rSiBd;pB7OdfW1ytxoR5b-#fnKU{^6H(U3>9^xxac@Hmu*~?maG^+#g84g_mhx zRZlY#m)!>wTzvT=0|~ZoHen1m+kxQFfuh0MZ2Qu?-ZUXAGE`ofe|d9y*Kja@3UI|Wm>kgp z{LG2xyONtZbf>fUz&g~;EIs387Gmz}3v-YkR=eQ{9OO`K6S@H{i}}P|O{AARZTv{R zH%08#gP#V2Xfb#f z*Q9zkn1%y)HXFo$b+>MIR{v>yiL?H58a@8@CPM1Un}0TR5g_9l`>uD!i}PkzrlKby zj%oF_ZgkHY+VbX`qMIp6ONtg^JFB9p*^zFsXbVZHVS4G}#+h z+>v{q9Z;UJS=f7v%y#Q`{3eRFi6YqQQ75rRVeSiJD&#=;QI8% zxli&6!x#G=v=>*hAKbIRHnT*pCO9s}p&VR%_`;z^5NXBAVRZhWb+Se)80?&GbO}r$ z7aVShL=xWwl1OFPoba%@!nADn`p4;dYv+~9g$%7&@qp|S7aeaOX6~Av6q2KA>F~w{ zQ`wkzL|WkpPyXZc{NLyRMoVcina;m4k)DdgGNVRj)YO~M-Eyy@$hM`qW&uO^4Vd;U z7nzaNi`<`rjfL#1n=l+)p@~?b9{G*4!i&OX|I&>fL43oG?5`xQgdzM1)Ga93x7a_C zRP^E1v&pZ#`s}8@X7ltR>ngS{DlIj}8)7f4zAvZA9b>#~2BT-jmn0L*+^nADW+Am9 z3Z9n899g}9$#fOOH7&r?_GDn`8znEwkeK66p5ky*z5Tg{!(}Rg*256JGsHV;Eh^20 z;F^lLSm57<<<^Ksh;26wi{!Iuwg-M6jL2lH_R`I&dPuCnBu-0f)#XvxtqCEb6SZ%8 zk+#)dYE!{e(QF5TvxrK)35E3&1Dm4dBUa+`_z|ubb5YBu%tcU`l@)qH_@s!*Vk&e) zzie-|AI7!$-6n#rtLt%N@caADF29o?$yRT+kxme2vW_wK^hs~>mN(p)PUzbc+XPyu zr~rlvF}tRl+hTz$g>@I)Y)@@vbdxSlV^N9>y~v9nidu2i?n_N&@HCxwcLgd^LFT>R zB%l0ZPq^+g#al_Mrlg|>?m7aZ3Z8IYH4-%&u^*jrwb*P&ze+ZHwfnobKmX7D%Nmrq4l1Z2Hq|@i#uxcL&Z^n9vEb4TXpR+eRO@sVx4> z`NQ@5n}_R<__LB8lN-2sj=aV5d~&;(O%c)E{pj-I>-f*lm)GNqKQ6AmT)h8sxoa;+ zoo#XxJv{$53&wXyLmS`0akLndt16%~VKv2BH>5l>hQ!~TJ9~;c<7C?^v5N&rQlTj4 z4e^Y@)CCsP#TsXuZ62U)G@mY)!NyB-79r&UZLFu|xvHHg@I+HIb)agehI$Ik!NdIr zv=vg4f1p3m89LkSL$r$*izJsV{?pG#oFv&#Nb;a#7w077tR?(g@{x7{D(C= 1: + logging.info(f"Waited for time {i*5} seconds") + break - # Run nifi ingestion run. - pipeline = Pipeline.create( - { - "run_id": "nifi-test-cluster", - "source": { - "type": "nifi", - "config": { - "site_url": "http://localhost:9080/nifi/", - "auth": "NO_AUTH", - "site_url_to_site_name": { - "http://nifi01:9080/nifi/": "default", - "http://nifi02:9081/nifi/": "default", - }, - }, + # Run the metadata ingestion pipeline. + with fs_helpers.isolated_filesystem(tmp_path): + # Run nifi ingestion run. + pipeline = Pipeline.create( + { + "run_id": "nifi-test-standalone", + "source": { + "type": "nifi", + "config": { + "site_url": "http://localhost:9443/nifi/", + "process_group_pattern": {"deny": ["^WIP"]}, }, - "sink": { - "type": "file", - "config": {"filename": "./nifi_mces_cluster.json"}, + }, + "sink": { + "type": "file", + "config": {"filename": "./nifi_mces.json"}, + }, + } + ) + pipeline.run() + pipeline.raise_from_status() + + # Verify the output. ignore values for aspects having last_event_time values + mce_helpers.check_golden_file( + pytestconfig, + output_path="nifi_mces.json", + golden_path=test_resources_dir / "nifi_mces_golden_standalone.json", + ignore_paths=[ + *mce_helpers.IGNORE_PATH_TIMESTAMPS, + r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['last_event_time'\]", + ], + ) + + +@freeze_time(FROZEN_TIME) +@pytest.mark.slow_integration +def test_nifi_ingest_cluster(loaded_nifi, pytestconfig, test_resources_dir): + # Wait for nifi cluster to execute all lineage processors, max wait time 120 seconds + url = "http://localhost:9080/nifi-api/flow/process-groups/root" + for i in range(23): + logging.info("waiting...") + time.sleep(5) + resp = requests.get(url) + if resp.status_code != 200: + continue + else: + pgs = resp.json()["processGroupFlow"]["flow"]["processGroups"] + statuses = [pg["status"] for pg in pgs] + status = next(s for s in statuses if s["name"] == "Cluster_Site_S3_to_S3") + if status["aggregateSnapshot"]["flowFilesSent"] >= 1: + logging.info(f"Waited for time {i*5} seconds") + break + test_resources_dir = pytestconfig.rootpath / "tests/integration/nifi" + # Run nifi ingestion run. + pipeline = Pipeline.create( + { + "run_id": "nifi-test-cluster", + "source": { + "type": "nifi", + "config": { + "site_url": "http://localhost:9080/nifi/", + "auth": "NO_AUTH", + "site_url_to_site_name": { + "http://nifi01:9080/nifi/": "default", + "http://nifi02:9081/nifi/": "default", + "http://nifi03:9082/nifi/": "default", }, - } - ) - pipeline.run() - pipeline.raise_from_status() + }, + }, + "sink": { + "type": "file", + "config": {"filename": "./nifi_mces_cluster.json"}, + }, + } + ) + pipeline.run() + pipeline.raise_from_status() - # Verify the output. - mce_helpers.check_golden_file( - pytestconfig, - output_path="nifi_mces_cluster.json", - golden_path=test_resources_dir / "nifi_mces_golden_cluster.json", - ignore_paths=[ - r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['last_event_time'\]", - ], - ) + # Verify the output. + mce_helpers.check_golden_file( + pytestconfig, + output_path="nifi_mces_cluster.json", + golden_path=test_resources_dir / "nifi_mces_golden_cluster.json", + ignore_paths=[ + r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['last_event_time'\]", + ], + ) From 8617e072fa510cd8a3eb0ccbf5014cfad521757d Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Fri, 7 Jul 2023 16:18:52 -0400 Subject: [PATCH 110/222] build(ingest): Pin pydeequ to unblock CI (#8381) --- metadata-ingestion/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 613da006663ee..688a9f7a43ef5 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -227,7 +227,7 @@ def get_long_description(): } data_lake_profiling = { - "pydeequ>=1.0.1", + "pydeequ>=1.0.1, <1.1", "pyspark==3.0.3", } From 1f84bf5b2b15c8d9d2d984860e9a54126afd2bfa Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Fri, 7 Jul 2023 19:05:50 -0400 Subject: [PATCH 111/222] fix(ingest/sql-common): Fix profile_table_level_only (#8331) Co-authored-by: Aseem Bansal Co-authored-by: Tamas Nemeth Co-authored-by: Harshal Sheth --- docs/how/updating-datahub.md | 4 + .../ingestion/source/ge_data_profiler.py | 17 +- .../ingestion/source/ge_profiling_config.py | 8 +- .../mysql/mysql_profile_table_level_only.yml | 14 + .../mysql/mysql_table_level_only.json | 393 ++++++++++++++++++ .../tests/integration/mysql/test_mysql.py | 37 +- ..._db_to_file_with_db_estimate_row_count.yml | 1 - .../postgres_mces_with_db_golden.json | 60 +-- ...res_to_file_with_db_estimate_row_count.yml | 12 - .../tests/unit/test_ge_profiling_config.py | 33 ++ 10 files changed, 477 insertions(+), 102 deletions(-) create mode 100644 metadata-ingestion/tests/integration/mysql/mysql_profile_table_level_only.yml create mode 100644 metadata-ingestion/tests/integration/mysql/mysql_table_level_only.json create mode 100644 metadata-ingestion/tests/unit/test_ge_profiling_config.py diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 3f38c5bc2e764..2123b07d166d4 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -10,6 +10,10 @@ This file documents any backwards-incompatible changes in DataHub and assists pe - #8263: Okta source config option `okta_profile_to_username_attr` default changed from `login` to `email`. This determines which Okta profile attribute is used for the corresponding DataHub user and thus may change what DataHub users are generated by the Okta source. And in a follow up `okta_profile_to_username_regex` has been set to `.*` which taken together with previous change brings the defaults in line with OIDC. +- #8331: For all sql-based sources that support profiling, you can no longer specify +`profile_table_level_only` together with `include_field_xyz` config options to ingest +certain column-level metrics. Instead, set `profile_table_level_only` to `false` and +individually enable / disable desired field metrics. ### Potential Downtime diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py index 5404b31763d23..ab259b9b16dd2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py @@ -579,16 +579,17 @@ def generate_dataset_profile( # noqa: C901 (complexity) self.query_combiner.flush() columns_profiling_queue: List[_SingleColumnSpec] = [] - for column in all_columns: - column_profile = DatasetFieldProfileClass(fieldPath=column) - profile.fieldProfiles.append(column_profile) + if columns_to_profile: + for column in all_columns: + column_profile = DatasetFieldProfileClass(fieldPath=column) + profile.fieldProfiles.append(column_profile) - if column in columns_to_profile: - column_spec = _SingleColumnSpec(column, column_profile) - columns_profiling_queue.append(column_spec) + if column in columns_to_profile: + column_spec = _SingleColumnSpec(column, column_profile) + columns_profiling_queue.append(column_spec) - self._get_column_type(column_spec, column) - self._get_column_cardinality(column_spec, column) + self._get_column_type(column_spec, column) + self._get_column_cardinality(column_spec, column) logger.debug(f"profiling {self.dataset_name}: flushing stage 2 queries") self.query_combiner.flush() diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py b/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py index 7805bf3b9a503..3f61a99bfd585 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py @@ -156,7 +156,7 @@ def deprecate_bigquery_temp_table_schema(cls, values): del values["bigquery_temp_table_schema"] return values - @pydantic.root_validator() + @pydantic.root_validator(pre=True) def ensure_field_level_settings_are_normalized( cls: "GEProfilingConfig", values: Dict[str, Any] ) -> Dict[str, Any]: @@ -167,7 +167,11 @@ def ensure_field_level_settings_are_normalized( if values.get("profile_table_level_only"): for field_level_metric in cls.__fields__: if field_level_metric.startswith("include_field_"): - values.setdefault(field_level_metric, False) + if values.get(field_level_metric): + raise ValueError( + "Cannot enable field-level metrics if profile_table_level_only is set" + ) + values[field_level_metric] = False assert ( max_num_fields_to_profile is None diff --git a/metadata-ingestion/tests/integration/mysql/mysql_profile_table_level_only.yml b/metadata-ingestion/tests/integration/mysql/mysql_profile_table_level_only.yml new file mode 100644 index 0000000000000..b3be63b57cf3c --- /dev/null +++ b/metadata-ingestion/tests/integration/mysql/mysql_profile_table_level_only.yml @@ -0,0 +1,14 @@ +source: + type: mysql + config: + username: root + password: example + host_port: localhost:53307 + database: northwind + profiling: + enabled: True + profile_table_level_only: true +sink: + type: file + config: + filename: "./mysql_mces.json" diff --git a/metadata-ingestion/tests/integration/mysql/mysql_table_level_only.json b/metadata-ingestion/tests/integration/mysql/mysql_table_level_only.json new file mode 100644 index 0000000000000..8b266a91bad06 --- /dev/null +++ b/metadata-ingestion/tests/integration/mysql/mysql_table_level_only.json @@ -0,0 +1,393 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "mysql", + "env": "PROD", + "database": "northwind" + }, + "name": "northwind" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mysql" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Database" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "customers", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "northwind.customers", + "platform": "urn:li:dataPlatform:mysql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "company", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=50)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=50)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=50)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email_address", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=50)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "orders", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "northwind.orders", + "platform": "urn:li:dataPlatform:mysql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "description", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=50)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + } + ], + "foreignKeys": [ + { + "name": "fk_order_customer", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),id)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),customer_id)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1586847600000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 5, + "columnCount": 6, + "fieldProfiles": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1586847600000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 0, + "columnCount": 3, + "fieldProfiles": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/mysql/test_mysql.py b/metadata-ingestion/tests/integration/mysql/test_mysql.py index 30c1bc1f70a10..bf5806bc57b13 100644 --- a/metadata-ingestion/tests/integration/mysql/test_mysql.py +++ b/metadata-ingestion/tests/integration/mysql/test_mysql.py @@ -42,37 +42,34 @@ def mysql_runner(docker_compose_runner, pytestconfig, test_resources_dir): yield docker_services +@pytest.mark.parametrize( + "config_file,golden_file", + [ + ("mysql_to_file_with_db.yml", "mysql_mces_with_db_golden.json"), + ("mysql_to_file_no_db.yml", "mysql_mces_no_db_golden.json"), + ("mysql_profile_table_level_only.yml", "mysql_table_level_only.json"), + ], +) @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_mysql_ingest_no_db( - mysql_runner, pytestconfig, test_resources_dir, tmp_path, mock_time -): - # Run the metadata ingestion pipeline. - config_file = (test_resources_dir / "mysql_to_file_no_db.yml").resolve() - run_datahub_cmd(["ingest", "-c", f"{config_file}"], tmp_path=tmp_path) - - # Verify the output. - mce_helpers.check_golden_file( - pytestconfig, - output_path=tmp_path / "mysql_mces.json", - golden_path=test_resources_dir / "mysql_mces_no_db_golden.json", - ) - - -@freeze_time(FROZEN_TIME) -@pytest.mark.integration -def test_mysql_ingest_with_db( - mysql_runner, pytestconfig, test_resources_dir, tmp_path, mock_time + mysql_runner, + pytestconfig, + test_resources_dir, + tmp_path, + mock_time, + config_file, + golden_file, ): # Run the metadata ingestion pipeline. - config_file = (test_resources_dir / "mysql_to_file_with_db.yml").resolve() + config_file = (test_resources_dir / config_file).resolve() run_datahub_cmd(["ingest", "-c", f"{config_file}"], tmp_path=tmp_path) # Verify the output. mce_helpers.check_golden_file( pytestconfig, output_path=tmp_path / "mysql_mces.json", - golden_path=test_resources_dir / "mysql_mces_with_db_golden.json", + golden_path=test_resources_dir / golden_file, ) diff --git a/metadata-ingestion/tests/integration/postgres/postgres_all_db_to_file_with_db_estimate_row_count.yml b/metadata-ingestion/tests/integration/postgres/postgres_all_db_to_file_with_db_estimate_row_count.yml index bfbad5fe83b6c..b390d9246677e 100644 --- a/metadata-ingestion/tests/integration/postgres/postgres_all_db_to_file_with_db_estimate_row_count.yml +++ b/metadata-ingestion/tests/integration/postgres/postgres_all_db_to_file_with_db_estimate_row_count.yml @@ -10,7 +10,6 @@ source: include_view_lineage: True profiling: enabled: true - profile_table_level_only: true profile_table_row_count_estimate_only: true turn_off_expensive_profiling_metrics: true include_field_null_count: true diff --git a/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json b/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json index b8b7cd9bf3900..bf36a39a8c103 100644 --- a/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json +++ b/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json @@ -386,65 +386,7 @@ }, "rowCount": 2, "columnCount": 9, - "fieldProfiles": [ - { - "fieldPath": "urn", - "uniqueCount": 1, - "uniqueProportion": 0.5, - "nullCount": 0, - "nullProportion": 0.0 - }, - { - "fieldPath": "aspect", - "uniqueCount": 2, - "uniqueProportion": 1, - "nullCount": 0, - "nullProportion": 0.0 - }, - { - "fieldPath": "version", - "uniqueCount": 1, - "uniqueProportion": 0.5, - "nullCount": 0, - "nullProportion": 0.0 - }, - { - "fieldPath": "metadata", - "uniqueCount": 2, - "uniqueProportion": 1, - "nullCount": 0, - "nullProportion": 0.0 - }, - { - "fieldPath": "systemmetadata", - "uniqueCount": 0, - "nullCount": 2, - "nullProportion": 1 - }, - { - "fieldPath": "createdon", - "uniqueCount": 1, - "uniqueProportion": 0.5, - "nullCount": 0, - "nullProportion": 0.0 - }, - { - "fieldPath": "createdby", - "uniqueCount": 1, - "uniqueProportion": 0.5, - "nullCount": 0, - "nullProportion": 0.0 - }, - { - "fieldPath": "createdfor", - "uniqueCount": 0, - "nullCount": 2, - "nullProportion": 1 - }, - { - "fieldPath": "metadata_json" - } - ] + "fieldProfiles": [] } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/postgres/postgres_to_file_with_db_estimate_row_count.yml b/metadata-ingestion/tests/integration/postgres/postgres_to_file_with_db_estimate_row_count.yml index 712ce4a2b69e4..a489877d52a23 100644 --- a/metadata-ingestion/tests/integration/postgres/postgres_to_file_with_db_estimate_row_count.yml +++ b/metadata-ingestion/tests/integration/postgres/postgres_to_file_with_db_estimate_row_count.yml @@ -12,18 +12,6 @@ source: profile_table_level_only: true profile_table_row_count_estimate_only: true turn_off_expensive_profiling_metrics: true - include_field_null_count: true - include_field_distinct_count: true - include_field_min_value: false - include_field_max_value: false - include_field_mean_value: false - include_field_median_value: false - include_field_stddev_value: false - include_field_sample_values: false - partition_profiling_enabled: false - include_field_quantiles: false - include_field_distinct_value_frequencies: false - include_field_histogram: false catch_exceptions: true include_views: false sink: diff --git a/metadata-ingestion/tests/unit/test_ge_profiling_config.py b/metadata-ingestion/tests/unit/test_ge_profiling_config.py new file mode 100644 index 0000000000000..f4d73a6ffe1e4 --- /dev/null +++ b/metadata-ingestion/tests/unit/test_ge_profiling_config.py @@ -0,0 +1,33 @@ +import pytest + +from datahub.ingestion.source.ge_profiling_config import GEProfilingConfig + + +def test_profile_table_level_only(): + config = GEProfilingConfig.parse_obj( + {"enabled": True, "profile_table_level_only": True} + ) + assert config.any_field_level_metrics_enabled() is False + + config = GEProfilingConfig.parse_obj( + { + "enabled": True, + "profile_table_level_only": True, + "include_field_max_value": False, + } + ) + assert config.any_field_level_metrics_enabled() is False + + +def test_profile_table_level_only_fails_with_field_metric_enabled(): + with pytest.raises( + ValueError, + match="Cannot enable field-level metrics if profile_table_level_only is set", + ): + GEProfilingConfig.parse_obj( + { + "enabled": True, + "profile_table_level_only": True, + "include_field_max_value": True, + } + ) From 3e47b3d2284a03b35632adb7e01bf77bf3208d54 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Fri, 7 Jul 2023 16:24:35 -0700 Subject: [PATCH 112/222] feat(ingest): schema-aware SQL parsing for column-level lineage (#8334) --- metadata-ingestion/setup.py | 11 + .../src/datahub/cli/check_cli.py | 74 ++ .../src/datahub/ingestion/graph/client.py | 44 +- .../source/bigquery_v2/bigquery_audit.py | 9 +- .../src/datahub/testing/__init__.py | 6 + .../testing/check_sql_parser_result.py | 90 +++ .../utilities/file_backed_collections.py | 26 +- .../src/datahub/utilities/sqlglot_lineage.py | 754 ++++++++++++++++++ .../test_bigquery_create_view_with_cte.json | 61 ++ ..._bigquery_from_sharded_table_wildcard.json | 33 + .../test_bigquery_nested_subqueries.json | 33 + ..._bigquery_sharded_table_normalization.json | 33 + .../goldens/test_bigquery_unnest_columns.json | 90 +++ .../goldens/test_create_view_as_select.json | 42 + .../test_expand_select_star_basic.json | 129 +++ .../goldens/test_insert_as_select.json | 76 ++ .../goldens/test_merge_from_union.json | 11 + .../goldens/test_select_count.json | 21 + .../test_select_from_struct_subfields.json | 49 ++ .../goldens/test_select_from_union.json | 33 + .../sql_parsing/goldens/test_select_max.json | 25 + .../goldens/test_select_with_ctes.json | 34 + .../test_select_with_full_col_name.json | 33 + .../test_snowflake_column_normalization.json | 57 ++ .../test_snowflake_default_normalization.json | 88 ++ .../unit/sql_parsing/test_sqlglot_lineage.py | 479 +++++++++++ .../tests/unit/test_bigquery_source.py | 3 +- 27 files changed, 2334 insertions(+), 10 deletions(-) create mode 100644 metadata-ingestion/src/datahub/testing/__init__.py create mode 100644 metadata-ingestion/src/datahub/testing/check_sql_parser_result.py create mode 100644 metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_create_view_with_cte.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_from_sharded_table_wildcard.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_nested_subqueries.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_sharded_table_normalization.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_unnest_columns.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_view_as_select.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_expand_select_star_basic.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_insert_as_select.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_merge_from_union.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_count.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_struct_subfields.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_union.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_max.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_ctes.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_full_col_name.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_column_normalization.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_default_normalization.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 688a9f7a43ef5..87cd2bbbd1dba 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -133,6 +133,12 @@ def get_long_description(): "sqlparse==0.4.3", } +sqlglot_lib = { + # Using an Acryl fork of sqlglot. + # https://github.com/tobymao/sqlglot/compare/main...hsheth2:sqlglot:hsheth?expand=1 + "acryl-sqlglot==16.7.6.dev6", +} + aws_common = { # AWS Python SDK "boto3", @@ -269,6 +275,8 @@ def get_long_description(): "gql[requests]>=3.3.0", }, "great-expectations": sql_common | sqllineage_lib, + # Misc plugins. + "sql-parser": sqlglot_lib, # Source plugins # PyAthena is pinned with exact version because we use private method in PyAthena "athena": sql_common | {"PyAthena[SQLAlchemy]==2.4.1"}, @@ -276,7 +284,9 @@ def get_long_description(): "bigquery": sql_common | bigquery_common | { + # TODO: I doubt we need all three sql parsing libraries. *sqllineage_lib, + *sqlglot_lib, "sql_metadata", "sqlalchemy-bigquery>=1.4.1", "google-cloud-datacatalog-lineage==0.2.2", @@ -285,6 +295,7 @@ def get_long_description(): | bigquery_common | { *sqllineage_lib, + *sqlglot_lib, "sql_metadata", "sqlalchemy-bigquery>=1.4.1", }, # deprecated, but keeping the extra for backwards compatibility diff --git a/metadata-ingestion/src/datahub/cli/check_cli.py b/metadata-ingestion/src/datahub/cli/check_cli.py index 3293c4ac96137..5d34967d496cd 100644 --- a/metadata-ingestion/src/datahub/cli/check_cli.py +++ b/metadata-ingestion/src/datahub/cli/check_cli.py @@ -1,14 +1,21 @@ +import logging import shutil import tempfile +from typing import Optional import click from datahub import __package_name__ from datahub.cli.json_file import check_mce_file +from datahub.emitter.mce_builder import DEFAULT_ENV +from datahub.ingestion.graph.client import get_default_graph from datahub.ingestion.run.pipeline import Pipeline from datahub.ingestion.sink.sink_registry import sink_registry from datahub.ingestion.source.source_registry import source_registry from datahub.ingestion.transformer.transform_registry import transform_registry +from datahub.telemetry import telemetry + +logger = logging.getLogger(__name__) @click.group() @@ -28,6 +35,7 @@ def check() -> None: @click.option( "--unpack-mces", default=False, is_flag=True, help="Converts MCEs into MCPs" ) +@telemetry.with_telemetry() def metadata_file(json_file: str, rewrite: bool, unpack_mces: bool) -> None: """Check the schema of a metadata (MCE or MCP) JSON file.""" @@ -70,6 +78,7 @@ def metadata_file(json_file: str, rewrite: bool, unpack_mces: bool) -> None: default=False, help="Include extra information for each plugin.", ) +@telemetry.with_telemetry() def plugins(verbose: bool) -> None: """List the enabled ingestion plugins.""" @@ -87,3 +96,68 @@ def plugins(verbose: bool) -> None: click.echo( f"If a plugin is disabled, try running: pip install '{__package_name__}[]'" ) + + +@check.command() +@click.option( + "--sql", + type=str, + required=True, + help="The SQL query to parse", +) +@click.option( + "--platform", + type=str, + required=True, + help="The SQL dialect e.g. bigquery or snowflake", +) +@click.option( + "--platform-instance", + type=str, + help="The specific platform_instance the SQL query was run in", +) +@click.option( + "--env", + type=str, + default=DEFAULT_ENV, + help=f"The environment the SQL query was run in, defaults to {DEFAULT_ENV}", +) +@click.option( + "--default-db", + type=str, + help="The default database to use for unqualified table names", +) +@click.option( + "--default-schema", + type=str, + help="The default schema to use for unqualified table names", +) +@telemetry.with_telemetry() +def sql_lineage( + sql: str, + platform: str, + default_db: Optional[str], + default_schema: Optional[str], + platform_instance: Optional[str], + env: str, +) -> None: + """Parse the lineage of a SQL query. + + This performs schema-aware parsing in order to generate column-level lineage. + If the relevant tables are not in DataHub, this will be less accurate. + """ + + graph = get_default_graph() + + lineage = graph.parse_sql_lineage( + sql, + platform=platform, + platform_instance=platform_instance, + env=env, + default_db=default_db, + default_schema=default_schema, + ) + + logger.debug("Sql parsing debug info: %s", lineage.debug_info) + + click.echo(lineage.json(indent=4)) diff --git a/metadata-ingestion/src/datahub/ingestion/graph/client.py b/metadata-ingestion/src/datahub/ingestion/graph/client.py index c3fb5dd4e37a7..ba07ea70c9656 100644 --- a/metadata-ingestion/src/datahub/ingestion/graph/client.py +++ b/metadata-ingestion/src/datahub/ingestion/graph/client.py @@ -1,4 +1,5 @@ import enum +import functools import json import logging import textwrap @@ -16,7 +17,7 @@ from datahub.configuration.common import ConfigModel, GraphError, OperationalError from datahub.configuration.validate_field_removal import pydantic_removed_field from datahub.emitter.aspect import TIMESERIES_ASPECT_MAP -from datahub.emitter.mce_builder import Aspect, make_data_platform_urn +from datahub.emitter.mce_builder import DEFAULT_ENV, Aspect, make_data_platform_urn from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.rest_emitter import DatahubRestEmitter from datahub.emitter.serialization_helper import post_json_transform @@ -44,6 +45,7 @@ from datahub.ingestion.source.state.entity_removal_state import ( GenericCheckpointState, ) + from datahub.utilities.sqlglot_lineage import SchemaResolver, SqlParsingResult logger = logging.getLogger(__name__) @@ -955,6 +957,46 @@ def delete_references_to_urn( related_aspects = response.get("relatedAspects", []) return reference_count, related_aspects + @functools.lru_cache() + def _make_schema_resolver( + self, platform: str, platform_instance: Optional[str], env: str + ) -> "SchemaResolver": + from datahub.utilities.sqlglot_lineage import SchemaResolver + + return SchemaResolver( + platform=platform, + platform_instance=platform_instance, + env=env, + graph=self, + ) + + def parse_sql_lineage( + self, + sql: str, + *, + platform: str, + platform_instance: Optional[str] = None, + env: str = DEFAULT_ENV, + default_db: Optional[str] = None, + default_schema: Optional[str] = None, + ) -> "SqlParsingResult": + from datahub.utilities.sqlglot_lineage import sqlglot_lineage + + # Cache the schema resolver to make bulk parsing faster. + schema_resolver = self._make_schema_resolver( + platform=platform, + platform_instance=platform_instance, + env=env, + ) + + return sqlglot_lineage( + sql, + platform=platform, + schema_resolver=schema_resolver, + default_db=default_db, + default_schema=default_schema, + ) + def get_default_graph() -> DataHubGraph: (url, token) = get_url_and_token() diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py index 379a773e248fa..0f9b37c93feaa 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py @@ -62,6 +62,8 @@ logger: logging.Logger = logging.getLogger(__name__) +_BIGQUERY_DEFAULT_SHARDED_TABLE_REGEX = "((.+)[_$])?(\\d{8})$" + @dataclass(frozen=True, order=True) class BigqueryTableIdentifier: @@ -70,7 +72,12 @@ class BigqueryTableIdentifier: table: str invalid_chars: ClassVar[Set[str]] = {"$", "@"} - _BIGQUERY_DEFAULT_SHARDED_TABLE_REGEX: ClassVar[str] = "((.+)[_$])?(\\d{8})$" + + # Note: this regex may get overwritten by the sharded_table_pattern config. + # The class-level constant, however, will not be overwritten. + _BIGQUERY_DEFAULT_SHARDED_TABLE_REGEX: ClassVar[ + str + ] = _BIGQUERY_DEFAULT_SHARDED_TABLE_REGEX _BIGQUERY_WILDCARD_REGEX: ClassVar[str] = "((_(\\d+)?)\\*$)|\\*$" _BQ_SHARDED_TABLE_SUFFIX: str = "_yyyymmdd" diff --git a/metadata-ingestion/src/datahub/testing/__init__.py b/metadata-ingestion/src/datahub/testing/__init__.py new file mode 100644 index 0000000000000..88eef332826bf --- /dev/null +++ b/metadata-ingestion/src/datahub/testing/__init__.py @@ -0,0 +1,6 @@ +"""Testing utilities for the datahub package. + +These modules are included in the `datahub.testing` namespace, but aren't +part of the public API. They're placed here, rather than in the `tests` +directory, so they can be used by other packages that depend on `datahub`. +""" diff --git a/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py b/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py new file mode 100644 index 0000000000000..b73af0478a1da --- /dev/null +++ b/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py @@ -0,0 +1,90 @@ +import logging +import os +import pathlib +from typing import Any, Dict, Optional + +import deepdiff + +from datahub.ingestion.source.bigquery_v2.bigquery_audit import BigqueryTableIdentifier +from datahub.utilities.sqlglot_lineage import ( + SchemaInfo, + SchemaResolver, + SqlParsingResult, + sqlglot_lineage, +) + +logger = logging.getLogger(__name__) + +# TODO: Hook this into the standard --update-golden-files mechanism. +UPDATE_FILES = os.environ.get("UPDATE_SQLPARSER_FILES", "false").lower() == "true" + + +def assert_sql_result_with_resolver( + sql: str, + *, + dialect: str, + expected_file: pathlib.Path, + schema_resolver: SchemaResolver, + **kwargs: Any, +) -> None: + # HACK: Our BigQuery source overwrites this value and doesn't undo it. + # As such, we need to handle that here. + BigqueryTableIdentifier._BQ_SHARDED_TABLE_SUFFIX = "_yyyymmdd" + + res = sqlglot_lineage( + sql, + platform=dialect, + schema_resolver=schema_resolver, + **kwargs, + ) + + if res.debug_info.column_error: + logger.warning( + f"SQL parser column error: {res.debug_info.column_error}", + exc_info=res.debug_info.column_error, + ) + + txt = res.json(indent=4) + if UPDATE_FILES: + expected_file.write_text(txt) + return + + if not expected_file.exists(): + expected_file.write_text(txt) + raise AssertionError( + f"Expected file {expected_file} does not exist. " + "Created it with the expected output. Please verify it." + ) + + expected = SqlParsingResult.parse_raw(expected_file.read_text()) + + full_diff = deepdiff.DeepDiff( + expected.dict(), + res.dict(), + exclude_regex_paths=[ + r"root.column_lineage\[\d+\].logic", + ], + ) + assert not full_diff, full_diff + + +def assert_sql_result( + sql: str, + *, + dialect: str, + expected_file: pathlib.Path, + schemas: Optional[Dict[str, SchemaInfo]] = None, + **kwargs: Any, +) -> None: + schema_resolver = SchemaResolver(platform=dialect) + if schemas: + for urn, schema in schemas.items(): + schema_resolver.add_raw_schema_info(urn, schema) + + assert_sql_result_with_resolver( + sql, + dialect=dialect, + expected_file=expected_file, + schema_resolver=schema_resolver, + **kwargs, + ) diff --git a/metadata-ingestion/src/datahub/utilities/file_backed_collections.py b/metadata-ingestion/src/datahub/utilities/file_backed_collections.py index 58e693b315653..74e97e5104105 100644 --- a/metadata-ingestion/src/datahub/utilities/file_backed_collections.py +++ b/metadata-ingestion/src/datahub/utilities/file_backed_collections.py @@ -55,18 +55,30 @@ class ConnectionWrapper: conn: sqlite3.Connection filename: pathlib.Path + _directory: Optional[tempfile.TemporaryDirectory] + _allow_table_name_reuse: bool def __init__(self, filename: Optional[pathlib.Path] = None): self._directory = None - # Warning: If filename is provided, the file will not be automatically cleaned up - if not filename: + + # In the normal case, we do not use "IF NOT EXISTS" in our create table statements + # because creating the same table twice indicates a client usage error. + # However, if you're trying to persist a file-backed dict across multiple runs, + # which happens when filename is passed explicitly, then we need to allow table name reuse. + allow_table_name_reuse = False + + # Warning: If filename is provided, the file will not be automatically cleaned up. + if filename: + allow_table_name_reuse = True + else: self._directory = tempfile.TemporaryDirectory() filename = pathlib.Path(self._directory.name) / _DEFAULT_FILE_NAME self.conn = sqlite3.connect(filename, isolation_level=None) self.conn.row_factory = sqlite3.Row self.filename = filename + self._allow_table_name_reuse = allow_table_name_reuse # These settings are optimized for performance. # See https://www.sqlite.org/pragma.html for more information. @@ -80,13 +92,13 @@ def __init__(self, filename: Optional[pathlib.Path] = None): def execute( self, sql: str, parameters: Union[Dict[str, Any], Sequence[Any]] = () ) -> sqlite3.Cursor: - logger.debug(f"Executing <{sql}> ({parameters})") + # logger.debug(f"Executing <{sql}> ({parameters})") return self.conn.execute(sql, parameters) def executemany( self, sql: str, parameters: Union[Dict[str, Any], Sequence[Any]] = () ) -> sqlite3.Cursor: - logger.debug(f"Executing many <{sql}> ({parameters})") + # logger.debug(f"Executing many <{sql}> ({parameters})") return self.conn.executemany(sql, parameters) def close(self) -> None: @@ -184,10 +196,10 @@ def __post_init__(self) -> None: # a poor-man's LRU cache. self._active_object_cache = collections.OrderedDict() - # Create the table. We're not using "IF NOT EXISTS" because creating - # the same table twice indicates a client usage error. + # Create the table. + if_not_exists = "IF NOT EXISTS" if self._conn._allow_table_name_reuse else "" self._conn.execute( - f"""CREATE TABLE {self.tablename} ( + f"""CREATE TABLE {if_not_exists} {self.tablename} ( key TEXT PRIMARY KEY, value BLOB {''.join(f', {column_name} BLOB' for column_name in self.extra_columns.keys())} diff --git a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py new file mode 100644 index 0000000000000..189c5bb269f67 --- /dev/null +++ b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py @@ -0,0 +1,754 @@ +import contextlib +import enum +import functools +import itertools +import logging +import pathlib +from collections import defaultdict +from typing import Dict, List, Optional, Set, Tuple, Union + +import pydantic +import pydantic.dataclasses +import sqlglot +import sqlglot.errors +import sqlglot.lineage +import sqlglot.optimizer.qualify +import sqlglot.optimizer.qualify_columns +from pydantic import BaseModel + +from datahub.emitter.mce_builder import ( + DEFAULT_ENV, + make_dataset_urn_with_platform_instance, +) +from datahub.ingestion.graph.client import DataHubGraph +from datahub.ingestion.source.bigquery_v2.bigquery_audit import BigqueryTableIdentifier +from datahub.metadata.schema_classes import SchemaMetadataClass +from datahub.utilities.file_backed_collections import ConnectionWrapper, FileBackedDict +from datahub.utilities.urns.dataset_urn import DatasetUrn + +logger = logging.getLogger(__name__) + +Urn = str + +# A lightweight table schema: column -> type mapping. +SchemaInfo = Dict[str, str] + + +class QueryType(enum.Enum): + CREATE = "CREATE" + SELECT = "SELECT" + INSERT = "INSERT" + UPDATE = "UPDATE" + DELETE = "DELETE" + MERGE = "MERGE" + + UNKNOWN = "UNKNOWN" + + +def get_query_type_of_sql(expression: sqlglot.exp.Expression) -> QueryType: + mapping = { + sqlglot.exp.Create: QueryType.CREATE, + sqlglot.exp.Select: QueryType.SELECT, + sqlglot.exp.Insert: QueryType.INSERT, + sqlglot.exp.Update: QueryType.UPDATE, + sqlglot.exp.Delete: QueryType.DELETE, + sqlglot.exp.Merge: QueryType.MERGE, + } + + for cls, query_type in mapping.items(): + if isinstance(expression, cls): + return query_type + return QueryType.UNKNOWN + + +@functools.total_ordering +class _FrozenModel(BaseModel, frozen=True): + def __lt__(self, other: "_FrozenModel") -> bool: + for field in self.__fields__: + self_v = getattr(self, field) + other_v = getattr(other, field) + if self_v != other_v: + return self_v < other_v + + return False + + +class _TableName(_FrozenModel): + database: Optional[str] + db_schema: Optional[str] + table: str + + def as_sqlglot_table(self) -> sqlglot.exp.Table: + return sqlglot.exp.Table( + catalog=self.database, db=self.db_schema, this=self.table + ) + + def qualified( + self, + dialect: str, + default_db: Optional[str] = None, + default_schema: Optional[str] = None, + ) -> "_TableName": + database = self.database or default_db + db_schema = self.db_schema or default_schema + + return _TableName( + database=database, + db_schema=db_schema, + table=self.table, + ) + + @classmethod + def from_sqlglot_table( + cls, + table: sqlglot.exp.Table, + dialect: str, + default_db: Optional[str] = None, + default_schema: Optional[str] = None, + ) -> "_TableName": + return cls( + database=table.catalog or default_db, + db_schema=table.db or default_schema, + table=table.this.name, + ) + + +class _ColumnRef(_FrozenModel): + table: _TableName + column: str + + +class ColumnRef(BaseModel): + table: Urn + column: str + + +class _DownstreamColumnRef(BaseModel): + table: Optional[_TableName] + column: str + + +class DownstreamColumnRef(BaseModel): + table: Optional[Urn] + column: str + + +class _ColumnLineageInfo(BaseModel): + downstream: _DownstreamColumnRef + upstreams: List[_ColumnRef] + + logic: Optional[str] + + +class ColumnLineageInfo(BaseModel): + downstream: DownstreamColumnRef + upstreams: List[ColumnRef] + + # Logic for this column, as a SQL expression. + logic: Optional[str] = pydantic.Field(default=None, exclude=True) + + +class SqlParsingDebugInfo(BaseModel, arbitrary_types_allowed=True): + confidence: float + + tables_discovered: int + table_schemas_resolved: int + + column_error: Optional[Exception] + + +class SqlParsingResult(BaseModel): + query_type: QueryType = QueryType.UNKNOWN + + in_tables: List[Urn] + out_tables: List[Urn] + + column_lineage: Optional[List[ColumnLineageInfo]] + + # TODO include formatted original sql logic + # TODO include list of referenced columns + + debug_info: SqlParsingDebugInfo = pydantic.Field( + default_factory=lambda: SqlParsingDebugInfo( + confidence=0, + tables_discovered=0, + table_schemas_resolved=0, + column_error=None, + ), + exclude=True, + ) + + +def _parse_statement(sql: str, dialect: str) -> sqlglot.Expression: + statement = sqlglot.parse_one( + sql, read=dialect, error_level=sqlglot.ErrorLevel.RAISE + ) + return statement + + +def _table_level_lineage( + statement: sqlglot.Expression, + dialect: str, +) -> Tuple[Set[_TableName], Set[_TableName]]: + def _raw_table_name(table: sqlglot.exp.Table) -> _TableName: + return _TableName.from_sqlglot_table(table, dialect=dialect) + + # Generate table-level lineage. + modified = { + _raw_table_name(expr.this) + for expr in statement.find_all( + sqlglot.exp.Create, + sqlglot.exp.Insert, + sqlglot.exp.Update, + sqlglot.exp.Delete, + sqlglot.exp.Merge, + ) + # In some cases like "MERGE ... then INSERT (col1, col2) VALUES (col1, col2)", + # the `this` on the INSERT part isn't a table. + if isinstance(expr.this, sqlglot.exp.Table) + } + + tables = ( + {_raw_table_name(table) for table in statement.find_all(sqlglot.exp.Table)} + # ignore references created in this query + - modified + # ignore CTEs created in this statement + - { + _TableName(database=None, schema=None, table=cte.alias_or_name) + for cte in statement.find_all(sqlglot.exp.CTE) + } + ) + # TODO: If a CTAS has "LIMIT 0", it's not really lineage, just copying the schema. + + return tables, modified + + +class SchemaResolver: + def __init__( + self, + *, + platform: str, + platform_instance: Optional[str] = None, + env: str = DEFAULT_ENV, + graph: Optional[DataHubGraph] = None, + _cache_filename: Optional[pathlib.Path] = None, + ): + # TODO handle platforms when prefixed with urn:li:dataPlatform: + self.platform = platform + self.platform_instance = platform_instance + self.env = env + + self.graph = graph + + # Init cache, potentially restoring from a previous run. + shared_conn = None + if _cache_filename: + shared_conn = ConnectionWrapper(filename=_cache_filename) + self._schema_cache: FileBackedDict[Optional[SchemaInfo]] = FileBackedDict( + shared_connection=shared_conn, + ) + + def get_urn_for_table(self, table: _TableName, lower: bool = False) -> str: + # TODO: Validate that this is the correct 2/3 layer hierarchy for the platform. + + table_name = ".".join( + filter(None, [table.database, table.db_schema, table.table]) + ) + if lower: + table_name = table_name.lower() + + if self.platform == "bigquery": + # Normalize shard numbers and other BigQuery weirdness. + # TODO check that this is the right way to do it + with contextlib.suppress(IndexError): + table_name = BigqueryTableIdentifier.from_string_name( + table_name + ).get_table_name() + + urn = make_dataset_urn_with_platform_instance( + platform=self.platform, + platform_instance=self.platform_instance, + env=self.env, + name=table_name, + ) + return urn + + def resolve_table(self, table: _TableName) -> Tuple[str, Optional[SchemaInfo]]: + urn = self.get_urn_for_table(table) + + schema_info = self._resolve_schema_info(urn) + if schema_info: + return urn, schema_info + + urn_lower = self.get_urn_for_table(table, lower=True) + if urn_lower != urn: + schema_info = self._resolve_schema_info(urn_lower) + if schema_info: + return urn_lower, schema_info + + return urn_lower, None + + def _resolve_schema_info(self, urn: str) -> Optional[SchemaInfo]: + if urn in self._schema_cache: + return self._schema_cache[urn] + + # TODO: For bigquery partitioned tables, add the pseudo-column _PARTITIONTIME + # or _PARTITIONDATE where appropriate. + + if self.graph: + schema_info = self._fetch_schema_info(self.graph, urn) + if schema_info: + self._save_to_cache(urn, schema_info) + return schema_info + + self._save_to_cache(urn, None) + return None + + def add_schema_metadata( + self, urn: str, schema_metadata: SchemaMetadataClass + ) -> None: + schema_info = self._convert_schema_aspect_to_info(schema_metadata) + self._save_to_cache(urn, schema_info) + + def add_raw_schema_info(self, urn: str, schema_info: SchemaInfo) -> None: + self._save_to_cache(urn, schema_info) + + def _save_to_cache(self, urn: str, schema_info: Optional[SchemaInfo]) -> None: + self._schema_cache[urn] = schema_info + + def _fetch_schema_info(self, graph: DataHubGraph, urn: str) -> Optional[SchemaInfo]: + aspect = graph.get_aspect(urn, SchemaMetadataClass) + if not aspect: + return None + + return self._convert_schema_aspect_to_info(aspect) + + @classmethod + def _convert_schema_aspect_to_info( + cls, schema_metadata: SchemaMetadataClass + ) -> SchemaInfo: + return { + DatasetUrn._get_simple_field_path_from_v2_field_path(col.fieldPath): ( + # The actual types are more of a "nice to have". + col.nativeDataType + or "str" + ) + for col in schema_metadata.fields + # TODO: We can't generate lineage to columns nested within structs yet. + if "." + not in DatasetUrn._get_simple_field_path_from_v2_field_path(col.fieldPath) + } + + # TODO add a method to load all from graphql + + +# TODO: Once PEP 604 is supported (Python 3.10), we can unify these into a +# single type. See https://peps.python.org/pep-0604/#isinstance-and-issubclass. +_SupportedColumnLineageTypes = Union[ + # Note that Select and Union inherit from Subqueryable. + sqlglot.exp.Subqueryable, + # For actual subqueries, the statement type might also be DerivedTable. + sqlglot.exp.DerivedTable, +] +_SupportedColumnLineageTypesTuple = (sqlglot.exp.Subqueryable, sqlglot.exp.DerivedTable) + + +class UnsupportedStatementTypeError(TypeError): + pass + + +class SqlUnderstandingError(Exception): + # Usually hit when we need schema info for a given statement but don't have it. + pass + + +# TODO: Break this up into smaller functions. +def _column_level_lineage( # noqa: C901 + statement: sqlglot.exp.Expression, + dialect: str, + input_tables: Dict[_TableName, SchemaInfo], + output_table: Optional[_TableName], + default_db: Optional[str], + default_schema: Optional[str], +) -> List[_ColumnLineageInfo]: + if not isinstance( + statement, + _SupportedColumnLineageTypesTuple, + ): + raise UnsupportedStatementTypeError( + f"Can only generate column-level lineage for select-like inner statements, not {type(statement)}" + ) + + use_case_insensitive_cols = dialect in { + # Column identifiers are case-insensitive in BigQuery, so we need to + # do a normalization step beforehand to make sure it's resolved correctly. + "bigquery", + # Our snowflake source lowercases column identifiers, so we are forced + # to do fuzzy (case-insensitive) resolution instead of exact resolution. + "snowflake", + } + + sqlglot_db_schema = sqlglot.MappingSchema( + dialect=dialect, + # We do our own normalization, so don't let sqlglot do it. + normalize=False, + ) + table_schema_normalized_mapping: Dict[_TableName, Dict[str, str]] = defaultdict( + dict + ) + for table, table_schema in input_tables.items(): + normalized_table_schema: SchemaInfo = {} + for col, col_type in table_schema.items(): + if use_case_insensitive_cols: + col_normalized = ( + # This is required to match Sqlglot's behavior. + col.upper() + if dialect in {"snowflake"} + else col.lower() + ) + else: + col_normalized = col + + table_schema_normalized_mapping[table][col_normalized] = col + normalized_table_schema[col_normalized] = col_type + + sqlglot_db_schema.add_table( + table.as_sqlglot_table(), + column_mapping=normalized_table_schema, + ) + + if use_case_insensitive_cols: + + def _sqlglot_force_column_normalizer( + node: sqlglot.exp.Expression, dialect: "sqlglot.DialectType" = None + ) -> sqlglot.exp.Expression: + if isinstance(node, sqlglot.exp.Column): + node.this.set("quoted", False) + + return node + + # logger.debug( + # "Prior to case normalization sql %s", + # statement.sql(pretty=True, dialect=dialect), + # ) + statement = statement.transform( + _sqlglot_force_column_normalizer, dialect, copy=False + ) + # logger.debug( + # "Sql after casing normalization %s", + # statement.sql(pretty=True, dialect=dialect), + # ) + + # Optimize the statement + qualify column references. + logger.debug( + "Prior to qualification sql %s", statement.sql(pretty=True, dialect=dialect) + ) + try: + # Second time running qualify, this time with: + # - the select instead of the full outer statement + # - schema info + # - column qualification enabled + + # logger.debug("Schema: %s", sqlglot_db_schema.mapping) + statement = sqlglot.optimizer.qualify.qualify( + statement, + dialect=dialect, + schema=sqlglot_db_schema, + validate_qualify_columns=False, + identify=True, + # sqlglot calls the db -> schema -> table hierarchy "catalog", "db", "table". + catalog=default_db, + db=default_schema, + ) + except (sqlglot.errors.OptimizeError, ValueError) as e: + raise SqlUnderstandingError( + f"sqlglot failed to map columns to their source tables; likely missing/outdated table schema info: {e}" + ) from e + logger.debug("Qualified sql %s", statement.sql(pretty=True, dialect=dialect)) + + column_lineage = [] + + try: + assert isinstance(statement, _SupportedColumnLineageTypesTuple) + + # List output columns. + output_columns = [ + (select_col.alias_or_name, select_col) for select_col in statement.selects + ] + logger.debug("output columns: %s", [col[0] for col in output_columns]) + output_col: str + for output_col, original_col_expression in output_columns: + # print(f"output column: {output_col}") + if output_col == "*": + # If schema information is available, the * will be expanded to the actual columns. + # Otherwise, we can't process it. + continue + + if dialect == "bigquery" and output_col.lower() in { + "_partitiontime", + "_partitiondate", + }: + # These are not real columns, just a way to filter by partition. + # TODO: We should add these columns to the schema info instead. + # Once that's done, we should actually generate lineage for these + # if they appear in the output. + continue + + lineage_node = sqlglot.lineage.lineage( + output_col, + statement, + dialect=dialect, + schema=sqlglot_db_schema, + ) + # pathlib.Path("sqlglot.html").write_text( + # str(lineage_node.to_html(dialect=dialect)) + # ) + + # Generate SELECT lineage. + # Using a set here to deduplicate upstreams. + direct_col_upstreams: Set[_ColumnRef] = set() + for node in lineage_node.walk(): + if node.downstream: + # We only want the leaf nodes. + pass + + elif isinstance(node.expression, sqlglot.exp.Table): + table_ref = _TableName.from_sqlglot_table( + node.expression, dialect=dialect + ) + + # Parse the column name out of the node name. + # Sqlglot calls .sql(), so we have to do the inverse. + normalized_col = sqlglot.parse_one(node.name).this.name + if node.subfield: + normalized_col = f"{normalized_col}.{node.subfield}" + col = table_schema_normalized_mapping[table_ref].get( + normalized_col, normalized_col + ) + + direct_col_upstreams.add(_ColumnRef(table=table_ref, column=col)) + else: + # This branch doesn't matter. For example, a count(*) column would go here, and + # we don't get any column-level lineage for that. + pass + + # column_logic = lineage_node.source + + if output_col.startswith("_col_"): + # This is the format sqlglot uses for unnamed columns e.g. 'count(id)' -> 'count(id) AS _col_0' + # This is a bit jank since we're relying on sqlglot internals, but it seems to be + # the best way to do it. + output_col = original_col_expression.this.sql(dialect=dialect) + if not direct_col_upstreams: + logger.debug(f' "{output_col}" has no upstreams') + column_lineage.append( + _ColumnLineageInfo( + downstream=_DownstreamColumnRef( + table=output_table, column=output_col + ), + upstreams=sorted(direct_col_upstreams), + # logic=column_logic.sql(pretty=True, dialect=dialect), + ) + ) + + # TODO: Also extract referenced columns (e.g. non-SELECT lineage) + except (sqlglot.errors.OptimizeError, ValueError) as e: + raise SqlUnderstandingError( + f"sqlglot failed to compute some lineage: {e}" + ) from e + + return column_lineage + + +def _extract_select_from_create( + statement: sqlglot.exp.Create, +) -> sqlglot.exp.Expression: + # TODO: Validate that this properly includes WITH clauses in all dialects. + inner = statement.expression + + if inner: + return inner + else: + return statement + + +def _try_extract_select( + statement: sqlglot.exp.Expression, +) -> sqlglot.exp.Expression: + # Try to extract the core select logic from a more complex statement. + # If it fails, just return the original statement. + + if isinstance(statement, sqlglot.exp.Merge): + # TODO Need to map column renames in the expressions part of the statement. + # Likely need to use the named_selects attr. + statement = statement.args["using"] + if isinstance(statement, sqlglot.exp.Table): + # If we're querying a table directly, wrap it in a SELECT. + statement = sqlglot.exp.Select().select("*").from_(statement) + elif isinstance(statement, sqlglot.exp.Insert): + # TODO Need to map column renames in the expressions part of the statement. + statement = statement.expression + elif isinstance(statement, sqlglot.exp.Create): + # TODO May need to map column renames. + # Assumption: the output table is already captured in the modified tables list. + statement = _extract_select_from_create(statement) + + if isinstance(statement, sqlglot.exp.Subquery): + statement = statement.unnest() + + return statement + + +def _translate_internal_column_lineage( + table_name_urn_mapping: Dict[_TableName, str], + raw_column_lineage: _ColumnLineageInfo, +) -> ColumnLineageInfo: + downstream_urn = None + if raw_column_lineage.downstream.table: + downstream_urn = table_name_urn_mapping[raw_column_lineage.downstream.table] + return ColumnLineageInfo( + downstream=DownstreamColumnRef( + table=downstream_urn, + column=raw_column_lineage.downstream.column, + ), + upstreams=[ + ColumnRef( + table=table_name_urn_mapping[upstream.table], + column=upstream.column, + ) + for upstream in raw_column_lineage.upstreams + ], + logic=raw_column_lineage.logic, + ) + + +def sqlglot_lineage( + sql: str, + platform: str, + schema_resolver: SchemaResolver, + default_db: Optional[str] = None, + default_schema: Optional[str] = None, +) -> SqlParsingResult: + # TODO: convert datahub platform names to sqlglot dialect + dialect = platform + + if dialect == "snowflake": + # in snowflake, table identifiers must be uppercased to match sqlglot's behavior. + if default_db: + default_db = default_db.upper() + if default_schema: + default_schema = default_schema.upper() + + logger.debug("Parsing lineage from sql statement: %s", sql) + statement = _parse_statement(sql, dialect=dialect) + + original_statement = statement.copy() + # logger.debug( + # "Formatted sql statement: %s", + # original_statement.sql(pretty=True, dialect=dialect), + # ) + + # Make sure the tables are resolved with the default db / schema. + # This only works for Unionable statements. For other types of statements, + # we have to do it manually afterwards, but that's slightly lower accuracy + # because of CTEs. + statement = sqlglot.optimizer.qualify.qualify( + statement, + dialect=dialect, + # sqlglot calls the db -> schema -> table hierarchy "catalog", "db", "table". + catalog=default_db, + db=default_schema, + # At this stage we only want to qualify the table names. The columns will be dealt with later. + qualify_columns=False, + validate_qualify_columns=False, + # Only insert quotes where necessary. + identify=False, + ) + + # Generate table-level lineage. + tables, modified = _table_level_lineage(statement, dialect=dialect) + + # Prep for generating column-level lineage. + downstream_table: Optional[_TableName] = None + if len(modified) == 1: + downstream_table = next(iter(modified)) + + # Fetch schema info for the relevant tables. + table_name_urn_mapping: Dict[_TableName, str] = {} + table_name_schema_mapping: Dict[_TableName, SchemaInfo] = {} + for table, is_input in itertools.chain( + [(table, True) for table in tables], + [(table, False) for table in modified], + ): + # For select statements, qualification will be a no-op. For other statements, this + # is where the qualification actually happens. + qualified_table = table.qualified( + dialect=dialect, default_db=default_db, default_schema=default_schema + ) + + urn, schema_info = schema_resolver.resolve_table(qualified_table) + + table_name_urn_mapping[qualified_table] = urn + if is_input and schema_info: + table_name_schema_mapping[qualified_table] = schema_info + + # Also include the original, non-qualified table name in the urn mapping. + table_name_urn_mapping[table] = urn + + debug_info = SqlParsingDebugInfo( + confidence=0.9 if len(tables) == len(table_name_schema_mapping) + # If we're missing any schema info, our confidence will be in the 0.2-0.5 range depending + # on how many tables we were able to resolve. + else 0.2 + 0.3 * len(table_name_schema_mapping) / len(tables), + tables_discovered=len(tables), + table_schemas_resolved=len(table_name_schema_mapping), + ) + logger.debug( + f"Resolved {len(table_name_schema_mapping)} of {len(tables)} table schemas" + ) + + # Simplify the input statement for column-level lineage generation. + select_statement = _try_extract_select(statement) + + # Generate column-level lineage. + column_lineage: Optional[List[_ColumnLineageInfo]] = None + try: + column_lineage = _column_level_lineage( + select_statement, + dialect=dialect, + input_tables=table_name_schema_mapping, + output_table=downstream_table, + default_db=default_db, + default_schema=default_schema, + ) + except UnsupportedStatementTypeError as e: + # Inject details about the outer statement type too. + e.args = (f"{e.args[0]} (outer statement type: {type(statement)})",) + debug_info.column_error = e + logger.debug(debug_info.column_error) + except SqlUnderstandingError as e: + logger.debug(f"Failed to generate column-level lineage: {e}", exc_info=True) + debug_info.column_error = e + + # TODO: Can we generate a common JOIN tables / keys section? + # TODO: Can we generate a common WHERE clauses section? + + # Convert TableName to urns. + in_urns = sorted(set(table_name_urn_mapping[table] for table in tables)) + out_urns = sorted(set(table_name_urn_mapping[table] for table in modified)) + column_lineage_urns = None + if column_lineage: + column_lineage_urns = [ + _translate_internal_column_lineage( + table_name_urn_mapping, internal_col_lineage + ) + for internal_col_lineage in column_lineage + ] + + return SqlParsingResult( + query_type=get_query_type_of_sql(original_statement), + in_tables=in_urns, + out_tables=out_urns, + column_lineage=column_lineage_urns, + debug_info=debug_info, + ) diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_create_view_with_cte.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_create_view_with_cte.json new file mode 100644 index 0000000000000..e50d944ce72e3 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_create_view_with_cte.json @@ -0,0 +1,61 @@ +{ + "query_type": "CREATE", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj-1.dataset.table1,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj-1.dataset.table2,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj-1.dataset.table3,PROD)" + ], + "out_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj-2.dataset.my_view,PROD)" + ], + "column_lineage": [ + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj-2.dataset.my_view,PROD)", + "column": "col5" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj-1.dataset.table3,PROD)", + "column": "col5" + } + ] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj-2.dataset.my_view,PROD)", + "column": "col1" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj-1.dataset.table1,PROD)", + "column": "col1" + } + ] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj-2.dataset.my_view,PROD)", + "column": "col2" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj-1.dataset.table1,PROD)", + "column": "col2" + } + ] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj-2.dataset.my_view,PROD)", + "column": "col3" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj-1.dataset.table2,PROD)", + "column": "col3" + } + ] + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_from_sharded_table_wildcard.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_from_sharded_table_wildcard.json new file mode 100644 index 0000000000000..78591286feb50 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_from_sharded_table_wildcard.json @@ -0,0 +1,33 @@ +{ + "query_type": "SELECT", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,bq-proj.dataset.table_yyyymmdd,PROD)" + ], + "out_tables": [], + "column_lineage": [ + { + "downstream": { + "table": null, + "column": "col1" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,bq-proj.dataset.table_yyyymmdd,PROD)", + "column": "col1" + } + ] + }, + { + "downstream": { + "table": null, + "column": "col2" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,bq-proj.dataset.table_yyyymmdd,PROD)", + "column": "col2" + } + ] + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_nested_subqueries.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_nested_subqueries.json new file mode 100644 index 0000000000000..0e93d31fbb6a6 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_nested_subqueries.json @@ -0,0 +1,33 @@ +{ + "query_type": "SELECT", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,bq-proj.dataset.table1,PROD)" + ], + "out_tables": [], + "column_lineage": [ + { + "downstream": { + "table": null, + "column": "col1" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,bq-proj.dataset.table1,PROD)", + "column": "col1" + } + ] + }, + { + "downstream": { + "table": null, + "column": "col2" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,bq-proj.dataset.table1,PROD)", + "column": "col2" + } + ] + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_sharded_table_normalization.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_sharded_table_normalization.json new file mode 100644 index 0000000000000..78591286feb50 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_sharded_table_normalization.json @@ -0,0 +1,33 @@ +{ + "query_type": "SELECT", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,bq-proj.dataset.table_yyyymmdd,PROD)" + ], + "out_tables": [], + "column_lineage": [ + { + "downstream": { + "table": null, + "column": "col1" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,bq-proj.dataset.table_yyyymmdd,PROD)", + "column": "col1" + } + ] + }, + { + "downstream": { + "table": null, + "column": "col2" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,bq-proj.dataset.table_yyyymmdd,PROD)", + "column": "col2" + } + ] + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_unnest_columns.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_unnest_columns.json new file mode 100644 index 0000000000000..69eb6f4ea631b --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_unnest_columns.json @@ -0,0 +1,90 @@ +{ + "query_type": "SELECT", + "in_tables": [ + { + "database": "bq-proj", + "db_schema": "dataset", + "table": "table1" + }, + { + "database": "bq-proj", + "db_schema": "dataset", + "table": "table2" + } + ], + "out_tables": [], + "column_lineage": [ + { + "downstream": { + "table": null, + "column": "day" + }, + "upstreams": [ + { + "table": { + "database": "bq-proj", + "db_schema": "dataset", + "table": "table1" + }, + "column": "reporting_day" + } + ] + }, + { + "downstream": { + "table": null, + "column": "product" + }, + "upstreams": [ + { + "table": { + "database": "bq-proj", + "db_schema": "dataset", + "table": "table1" + }, + "column": "by_product.product_code" + }, + { + "table": { + "database": "bq-proj", + "db_schema": "dataset", + "table": "table2" + }, + "column": "other_field" + } + ] + }, + { + "downstream": { + "table": null, + "column": "other_field" + }, + "upstreams": [ + { + "table": { + "database": "bq-proj", + "db_schema": "dataset", + "table": "table2" + }, + "column": "other_field" + } + ] + }, + { + "downstream": { + "table": null, + "column": "daily_active_users" + }, + "upstreams": [ + { + "table": { + "database": "bq-proj", + "db_schema": "dataset", + "table": "table1" + }, + "column": "by_product.product_code_dau" + } + ] + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_view_as_select.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_view_as_select.json new file mode 100644 index 0000000000000..22bb78dc86dd9 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_create_view_as_select.json @@ -0,0 +1,42 @@ +{ + "query_type": "CREATE", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:oracle,scott.emp,PROD)" + ], + "out_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:oracle,vsal,PROD)" + ], + "column_lineage": [ + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:oracle,vsal,PROD)", + "column": "Department" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:oracle,scott.emp,PROD)", + "column": "deptno" + } + ] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:oracle,vsal,PROD)", + "column": "Employees" + }, + "upstreams": [] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:oracle,vsal,PROD)", + "column": "Salary" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:oracle,scott.emp,PROD)", + "column": "sal" + } + ] + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_expand_select_star_basic.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_expand_select_star_basic.json new file mode 100644 index 0000000000000..e456e4450c50a --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_expand_select_star_basic.json @@ -0,0 +1,129 @@ +{ + "query_type": "SELECT", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)" + ], + "out_tables": [], + "column_lineage": [ + { + "downstream": { + "table": null, + "column": "TOTAL_AGG" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)", + "column": "totalprice" + } + ] + }, + { + "downstream": { + "table": null, + "column": "ORDERKEY" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)", + "column": "orderkey" + } + ] + }, + { + "downstream": { + "table": null, + "column": "CUSTKEY" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)", + "column": "custkey" + } + ] + }, + { + "downstream": { + "table": null, + "column": "ORDERSTATUS" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)", + "column": "orderstatus" + } + ] + }, + { + "downstream": { + "table": null, + "column": "TOTALPRICE" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)", + "column": "totalprice" + } + ] + }, + { + "downstream": { + "table": null, + "column": "ORDERDATE" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)", + "column": "orderdate" + } + ] + }, + { + "downstream": { + "table": null, + "column": "ORDERPRIORITY" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)", + "column": "orderpriority" + } + ] + }, + { + "downstream": { + "table": null, + "column": "CLERK" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)", + "column": "clerk" + } + ] + }, + { + "downstream": { + "table": null, + "column": "SHIPPRIORITY" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)", + "column": "shippriority" + } + ] + }, + { + "downstream": { + "table": null, + "column": "COMMENT" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)", + "column": "comment" + } + ] + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_insert_as_select.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_insert_as_select.json new file mode 100644 index 0000000000000..d7264fd2db6b2 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_insert_as_select.json @@ -0,0 +1,76 @@ +{ + "query_type": "INSERT", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,catalog_returns,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:hive,catalog_sales,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:hive,customer_demographics,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:hive,date_dim,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:hive,household_demographics,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:hive,inventory,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:hive,item,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:hive,promotion,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:hive,warehouse,PROD)" + ], + "out_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,query72,PROD)" + ], + "column_lineage": [ + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:hive,query72,PROD)", + "column": "i_item_desc" + }, + "upstreams": [] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:hive,query72,PROD)", + "column": "w_warehouse_name" + }, + "upstreams": [] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:hive,query72,PROD)", + "column": "d_week_seq" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:hive,date_dim,PROD)", + "column": "d_week_seq" + } + ] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:hive,query72,PROD)", + "column": "no_promo" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:hive,promotion,PROD)", + "column": "p_promo_sk" + } + ] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:hive,query72,PROD)", + "column": "promo" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:hive,promotion,PROD)", + "column": "p_promo_sk" + } + ] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:hive,query72,PROD)", + "column": "total_cnt" + }, + "upstreams": [] + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_merge_from_union.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_merge_from_union.json new file mode 100644 index 0000000000000..ec8599353fdca --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_merge_from_union.json @@ -0,0 +1,11 @@ +{ + "query_type": "MERGE", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,demo-pipelines-stg.referrer.prep_from_ios,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:bigquery,demo-pipelines-stg.referrer.prep_from_web,PROD)" + ], + "out_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,demo-pipelines-stg.referrer.base_union,PROD)" + ], + "column_lineage": null +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_count.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_count.json new file mode 100644 index 0000000000000..9f6eeae46c294 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_count.json @@ -0,0 +1,21 @@ +{ + "query_type": "SELECT", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:mysql,something_prd.fact_complaint_snapshot,PROD)" + ], + "out_tables": [], + "column_lineage": [ + { + "downstream": { + "table": null, + "column": "COUNT(`fact_complaint_snapshot`.`etl_data_dt_id`)" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:mysql,something_prd.fact_complaint_snapshot,PROD)", + "column": "etl_data_dt_id" + } + ] + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_struct_subfields.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_struct_subfields.json new file mode 100644 index 0000000000000..109de96180422 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_struct_subfields.json @@ -0,0 +1,49 @@ +{ + "query_type": "SELECT", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-bq-proj.data_reporting.abcde_transformed,PROD)" + ], + "out_tables": [], + "column_lineage": [ + { + "downstream": { + "table": null, + "column": "post_id" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-bq-proj.data_reporting.abcde_transformed,PROD)", + "column": "post_id" + } + ] + }, + { + "downstream": { + "table": null, + "column": "id" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-bq-proj.data_reporting.abcde_transformed,PROD)", + "column": "widget.asset.id" + } + ] + }, + { + "downstream": { + "table": null, + "column": "min_metric" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-bq-proj.data_reporting.abcde_transformed,PROD)", + "column": "widget.metric.metricA" + }, + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-bq-proj.data_reporting.abcde_transformed,PROD)", + "column": "widget.metric.metric_b" + } + ] + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_union.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_union.json new file mode 100644 index 0000000000000..8e1fd453ce09d --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_from_union.json @@ -0,0 +1,33 @@ +{ + "query_type": "UNKNOWN", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf10.orders,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf100.orders,PROD)" + ], + "out_tables": [], + "column_lineage": [ + { + "downstream": { + "table": null, + "column": "LABEL" + }, + "upstreams": [] + }, + { + "downstream": { + "table": null, + "column": "TOTAL_AGG" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf10.orders,PROD)", + "column": "totalprice" + }, + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf100.orders,PROD)", + "column": "totalprice" + } + ] + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_max.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_max.json new file mode 100644 index 0000000000000..326c07d332c26 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_max.json @@ -0,0 +1,25 @@ +{ + "query_type": "SELECT", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:mysql,mytable,PROD)" + ], + "out_tables": [], + "column_lineage": [ + { + "downstream": { + "table": null, + "column": "max_col" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:mysql,mytable,PROD)", + "column": "col1" + }, + { + "table": "urn:li:dataset:(urn:li:dataPlatform:mysql,mytable,PROD)", + "column": "col2" + } + ] + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_ctes.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_ctes.json new file mode 100644 index 0000000000000..4647b27934a10 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_ctes.json @@ -0,0 +1,34 @@ +{ + "query_type": "SELECT", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:oracle,table1,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:oracle,table2,PROD)" + ], + "out_tables": [], + "column_lineage": [ + { + "downstream": { + "table": null, + "column": "col1" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:oracle,table1,PROD)", + "column": "col1" + } + ] + }, + { + "downstream": { + "table": null, + "column": "col3" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:oracle,table2,PROD)", + "column": "col3" + } + ] + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_full_col_name.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_full_col_name.json new file mode 100644 index 0000000000000..c12ad23b2f03b --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_select_with_full_col_name.json @@ -0,0 +1,33 @@ +{ + "query_type": "SELECT", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-bq-ProjectName.data_reporting.abcde_transformed,PROD)" + ], + "out_tables": [], + "column_lineage": [ + { + "downstream": { + "table": null, + "column": "post_id" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-bq-ProjectName.data_reporting.abcde_transformed,PROD)", + "column": "post_id" + } + ] + }, + { + "downstream": { + "table": null, + "column": "id" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-bq-ProjectName.data_reporting.abcde_transformed,PROD)", + "column": "widget.asset.id" + } + ] + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_column_normalization.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_column_normalization.json new file mode 100644 index 0000000000000..694bec3800dbf --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_column_normalization.json @@ -0,0 +1,57 @@ +{ + "query_type": "SELECT", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)" + ], + "out_tables": [], + "column_lineage": [ + { + "downstream": { + "table": null, + "column": "TOTAL_AGG" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)", + "column": "TotalPrice" + } + ] + }, + { + "downstream": { + "table": null, + "column": "TOTAL_AVG" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)", + "column": "TotalPrice" + } + ] + }, + { + "downstream": { + "table": null, + "column": "TOTAL_MIN" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)", + "column": "TotalPrice" + } + ] + }, + { + "downstream": { + "table": null, + "column": "TOTAL_MAX" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)", + "column": "TotalPrice" + } + ] + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_default_normalization.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_default_normalization.json new file mode 100644 index 0000000000000..157745854128f --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_default_normalization.json @@ -0,0 +1,88 @@ +{ + "query_type": "CREATE", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.customer_last_purchase_date,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.ecommerce.purchases,PROD)" + ], + "out_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.active_customer_ltv,PROD)" + ], + "column_lineage": [ + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.active_customer_ltv,PROD)", + "column": "USER_FK" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.customer_last_purchase_date,PROD)", + "column": "USER_FK" + } + ] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.active_customer_ltv,PROD)", + "column": "EMAIL" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.customer_last_purchase_date,PROD)", + "column": "EMAIL" + } + ] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.active_customer_ltv,PROD)", + "column": "LAST_PURCHASE_DATE" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.customer_last_purchase_date,PROD)", + "column": "LAST_PURCHASE_DATE" + } + ] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.active_customer_ltv,PROD)", + "column": "LIFETIME_PURCHASE_AMOUNT" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.ecommerce.purchases,PROD)", + "column": "purchase_amount" + } + ] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.active_customer_ltv,PROD)", + "column": "LIFETIME_PURCHASE_COUNT" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.ecommerce.purchases,PROD)", + "column": "pk" + } + ] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.active_customer_ltv,PROD)", + "column": "AVERAGE_PURCHASE_AMOUNT" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.ecommerce.purchases,PROD)", + "column": "pk" + }, + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.ecommerce.purchases,PROD)", + "column": "purchase_amount" + } + ] + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py new file mode 100644 index 0000000000000..b7ef62ba82296 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py @@ -0,0 +1,479 @@ +import pathlib + +import pytest + +from datahub.testing.check_sql_parser_result import assert_sql_result + +RESOURCE_DIR = pathlib.Path(__file__).parent / "goldens" + + +def test_select_max(): + # The COL2 should get normalized to col2. + assert_sql_result( + """ +SELECT max(col1, COL2) as max_col +FROM mytable +""", + dialect="mysql", + expected_file=RESOURCE_DIR / "test_select_max.json", + ) + + +def test_select_max_with_schema(): + # Note that `this_will_not_resolve` will be dropped from the result because it's not in the schema. + assert_sql_result( + """ +SELECT max(`col1`, COL2, `this_will_not_resolve`) as max_col +FROM mytable +""", + dialect="mysql", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:mysql,mytable,PROD)": { + "col1": "NUMBER", + "col2": "NUMBER", + }, + }, + # Shared with the test above. + expected_file=RESOURCE_DIR / "test_select_max.json", + ) + + +def test_select_count(): + assert_sql_result( + """ +SELECT + COUNT(etl_data_dt_id) +FROM something_prd.fact_complaint_snapshot +WHERE + etl_data_dt_id = 20230317 +""", + dialect="mysql", + expected_file=RESOURCE_DIR / "test_select_count.json", + ) + + +def test_select_with_ctes(): + assert_sql_result( + """ +WITH cte1 AS ( + SELECT col1, col2 + FROM table1 + WHERE col1 = 'value1' +), cte2 AS ( + SELECT col3, col4 + FROM table2 + WHERE col2 = 'value2' +) +SELECT cte1.col1, cte2.col3 +FROM cte1 +JOIN cte2 ON cte1.col2 = cte2.col4 +""", + dialect="oracle", + expected_file=RESOURCE_DIR / "test_select_with_ctes.json", + ) + + +def test_create_view_as_select(): + assert_sql_result( + """ +CREATE VIEW vsal +AS + SELECT a.deptno "Department", + a.num_emp / b.total_count "Employees", + a.sal_sum / b.total_sal "Salary" + FROM (SELECT deptno, + Count() num_emp, + SUM(sal) sal_sum + FROM scott.emp + WHERE city = 'NYC' + GROUP BY deptno) a, + (SELECT Count() total_count, + SUM(sal) total_sal + FROM scott.emp + WHERE city = 'NYC') b +; +""", + dialect="oracle", + expected_file=RESOURCE_DIR / "test_create_view_as_select.json", + ) + + +def test_insert_as_select(): + # Note: this also tests lineage with case statements. + + assert_sql_result( + """ +insert into query72 +select i_item_desc + , w_warehouse_name + , d1.d_week_seq + , sum(case when promotion.p_promo_sk is null then 1 else 0 end) no_promo + , sum(case when promotion.p_promo_sk is not null then 1 else 0 end) promo + , count(*) total_cnt +from catalog_sales + join inventory on (cs_item_sk = inv_item_sk) + join warehouse on (w_warehouse_sk = inv_warehouse_sk) + join item on (i_item_sk = cs_item_sk) + join customer_demographics on (cs_bill_cdemo_sk = cd_demo_sk) + join household_demographics on (cs_bill_hdemo_sk = hd_demo_sk) + join date_dim d1 on (cs_sold_date_sk = d1.d_date_sk) + join date_dim d2 on (inv_date_sk = d2.d_date_sk) + join date_dim d3 on (cs_ship_date_sk = d3.d_date_sk) + left outer join promotion on (cs_promo_sk = p_promo_sk) + left outer join catalog_returns on (cr_item_sk = cs_item_sk and cr_order_number = cs_order_number) +where d1.d_week_seq = d2.d_week_seq + and inv_quantity_on_hand < cs_quantity + and hd_buy_potential = '>10000' + and cd_marital_status = 'D' +group by i_item_desc, w_warehouse_name, d1.d_week_seq +order by total_cnt desc, i_item_desc, w_warehouse_name, d_week_seq +limit 100; +""", + dialect="hive", + expected_file=RESOURCE_DIR / "test_insert_as_select.json", + ) + + +def test_select_with_full_col_name(): + # In this case, `widget` is a struct column. + # This also tests the `default_db` functionality. + assert_sql_result( + """ +SELECT distinct post_id , widget.asset.id +FROM data_reporting.abcde_transformed +WHERE post_id LIKE '%268662%' +""", + dialect="bigquery", + default_db="my-bq-ProjectName", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-bq-ProjectName.data_reporting.abcde_transformed,PROD)": { + "post_id": "NUMBER", + "widget": "struct", + }, + }, + expected_file=RESOURCE_DIR / "test_select_with_full_col_name.json", + ) + + +def test_select_from_struct_subfields(): + # In this case, `widget` is a column name. + assert_sql_result( + """ +SELECT distinct post_id , + widget.asset.id, + min(widget.metric.metricA, widget.metric.metric_b) as min_metric +FROM data_reporting.abcde_transformed +WHERE post_id LIKE '%12345%' +""", + dialect="bigquery", + default_db="my-bq-proj", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-bq-proj.data_reporting.abcde_transformed,PROD)": { + "post_id": "NUMBER", + "widget": "struct", + "widget.asset.id": "int", + "widget.metric.metricA": "int", + "widget.metric.metric_b": "int", + }, + }, + expected_file=RESOURCE_DIR / "test_select_from_struct_subfields.json", + ) + + +def test_select_from_union(): + assert_sql_result( + """ +SELECT + 'orders_10' as label, + SUM(totalprice) as total_agg +FROM snowflake_sample_data.tpch_sf10.orders +UNION ALL +SELECT + 'orders_100' as label, + SUM(totalprice) as total_agg, +FROM snowflake_sample_data.tpch_sf100.orders +""", + dialect="snowflake", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf10.orders,PROD)": { + "orderkey": "NUMBER", + "totalprice": "FLOAT", + }, + "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf100.orders,PROD)": { + "orderkey": "NUMBER", + "totalprice": "FLOAT", + }, + }, + expected_file=RESOURCE_DIR / "test_select_from_union.json", + ) + + +def test_merge_from_union(): + # TODO: We don't support merge statements yet, but the union should still get handled. + + assert_sql_result( + """ + merge into `demo-pipelines-stg`.`referrer`.`base_union` as DBT_INTERNAL_DEST + using ( +SELECT * FROM `demo-pipelines-stg`.`referrer`.`prep_from_ios` WHERE partition_time = "2018-03-03" +UNION ALL +SELECT * FROM `demo-pipelines-stg`.`referrer`.`prep_from_web` WHERE partition_time = "2018-03-03" + ) as DBT_INTERNAL_SOURCE + on FALSE + + when not matched by source + and timestamp_trunc(DBT_INTERNAL_DEST.partition_time, day) in ( + timestamp('2018-03-03') + ) + then delete + + when not matched then insert + (`platform`, `pageview_id`, `query`, `referrer`, `partition_time`) + values + (`platform`, `pageview_id`, `query`, `referrer`, `partition_time`) +""", + dialect="bigquery", + expected_file=RESOURCE_DIR / "test_merge_from_union.json", + ) + + +def test_expand_select_star_basic(): + assert_sql_result( + """ +SELECT + SUM(totalprice) as total_agg, + * +FROM snowflake_sample_data.tpch_sf1.orders +WHERE orderdate = '1992-01-01' +""", + dialect="snowflake", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)": { + "orderkey": "NUMBER", + "custkey": "NUMBER", + "orderstatus": "TEXT", + "totalprice": "FLOAT", + "orderdate": "DATE", + "orderpriority": "TEXT", + "clerk": "TEXT", + "shippriority": "NUMBER", + "comment": "TEXT", + }, + }, + expected_file=RESOURCE_DIR / "test_expand_select_star_basic.json", + ) + + +def test_snowflake_column_normalization(): + # Technically speaking this is incorrect since the column names are different and both quoted. + + assert_sql_result( + """ +SELECT + SUM(o."totalprice") as total_agg, + AVG("TotalPrice") as total_avg, + MIN("TOTALPRICE") as total_min, + MAX(TotalPrice) as total_max +FROM snowflake_sample_data.tpch_sf1.orders o +""", + dialect="snowflake", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)": { + "orderkey": "NUMBER", + "TotalPrice": "FLOAT", + }, + }, + expected_file=RESOURCE_DIR / "test_snowflake_column_normalization.json", + ) + + +@pytest.mark.skip(reason="We don't handle the unnest lineage correctly") +def test_bigquery_unnest_columns(): + assert_sql_result( + """ +SELECT + DATE(reporting_day) AS day, + CASE + WHEN p.product_code IN ('A', 'B', 'C') + THEN pr.other_field + ELSE 'Other' + END AS product, + pr.other_field AS other_field, + SUM(p.product_code_dau) AS daily_active_users +FROM `bq-proj`.dataset.table1 +LEFT JOIN UNNEST(by_product) AS p +LEFT JOIN ( + SELECT DISTINCT + product_code, + other_field + FROM `bq-proj`.dataset.table2 +) AS pr +-- USING (product_code) +ON p.product_code = pr.product_code +""", + dialect="bigquery", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,bq-proj.dataset.table1,PROD)": { + "reporting_day": "DATE", + "by_product": "ARRAY>", + }, + "urn:li:dataset:(urn:li:dataPlatform:bigquery,bq-proj.dataset.table2,PROD)": { + "product_code": "STRING", + "other_field": "STRING", + }, + }, + expected_file=RESOURCE_DIR / "test_bigquery_unnest_columns.json", + ) + + +def test_bigquery_create_view_with_cte(): + assert_sql_result( + """ +CREATE VIEW `my-proj-2`.dataset.my_view AS +WITH cte1 AS ( + SELECT * + FROM dataset.table1 + WHERE col1 = 'value1' +), cte2 AS ( + SELECT col3, col4 as join_key + FROM dataset.table2 + WHERE col3 = 'value2' +) +SELECT col5, cte1.*, col3 +FROM dataset.table3 +JOIN cte1 ON table3.col5 = cte1.col2 +JOIN cte2 USING (join_key) +""", + dialect="bigquery", + default_db="my-proj-1", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj-1.dataset.table1,PROD)": { + "col1": "STRING", + "col2": "STRING", + }, + "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj-1.dataset.table2,PROD)": { + "col3": "STRING", + "col4": "STRING", + }, + "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj-1.dataset.table3,PROD)": { + "col5": "STRING", + "join_key": "STRING", + }, + }, + expected_file=RESOURCE_DIR / "test_bigquery_create_view_with_cte.json", + ) + + +def test_bigquery_nested_subqueries(): + assert_sql_result( + """ +SELECT * +FROM ( + SELECT * + FROM ( + SELECT * + FROM `bq-proj`.dataset.table1 + ) +) +""", + dialect="bigquery", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,bq-proj.dataset.table1,PROD)": { + "col1": "STRING", + "col2": "STRING", + }, + }, + expected_file=RESOURCE_DIR / "test_bigquery_nested_subqueries.json", + ) + + +def test_bigquery_sharded_table_normalization(): + assert_sql_result( + """ +SELECT * +FROM `bq-proj.dataset.table_20230101` +""", + dialect="bigquery", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,bq-proj.dataset.table_yyyymmdd,PROD)": { + "col1": "STRING", + "col2": "STRING", + }, + }, + expected_file=RESOURCE_DIR / "test_bigquery_sharded_table_normalization.json", + ) + + +def test_bigquery_from_sharded_table_wildcard(): + assert_sql_result( + """ +SELECT * +FROM `bq-proj.dataset.table_2023*` +""", + dialect="bigquery", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,bq-proj.dataset.table_yyyymmdd,PROD)": { + "col1": "STRING", + "col2": "STRING", + }, + }, + expected_file=RESOURCE_DIR / "test_bigquery_from_sharded_table_wildcard.json", + ) + + +def test_snowflake_default_normalization(): + assert_sql_result( + """ +create table active_customer_ltv as ( + +with active_customers as ( + select * from customer_last_purchase_date + where + last_purchase_date >= current_date - interval '90 days' +) + +, purchases as ( + select * from ecommerce.purchases +) + +select + active_customers.user_fk + , active_customers.email + , active_customers.last_purchase_date + , sum(purchases.purchase_amount) as lifetime_purchase_amount + , count(distinct(purchases.pk)) as lifetime_purchase_count + , sum(purchases.purchase_amount) / count(distinct(purchases.pk)) as average_purchase_amount +from + active_customers +join + purchases + on active_customers.user_fk = purchases.user_fk +group by 1,2,3 + +) +""", + dialect="snowflake", + default_db="long_tail_companions", + default_schema="analytics", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.ecommerce.purchases,PROD)": { + "pk": "NUMBER(38,0)", + "USER_FK": "NUMBER(38,0)", + "status": "VARCHAR(16777216)", + "purchase_amount": "NUMBER(10,2)", + "tax_AMOUNT": "NUMBER(10,2)", + "TOTAL_AMOUNT": "NUMBER(10,2)", + "CREATED_AT": "TIMESTAMP_NTZ", + "UPDATED_AT": "TIMESTAMP_NTZ", + }, + "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.customer_last_purchase_date,PROD)": { + "USER_FK": "NUMBER(38,0)", + "EMAIL": "VARCHAR(16777216)", + "LAST_PURCHASE_DATE": "DATE", + }, + }, + expected_file=RESOURCE_DIR / "test_snowflake_default_normalization.json", + ) + + +# TODO: Add a test for setting platform_instance or env diff --git a/metadata-ingestion/tests/unit/test_bigquery_source.py b/metadata-ingestion/tests/unit/test_bigquery_source.py index 49dc66b2322f9..3efca8d088eed 100644 --- a/metadata-ingestion/tests/unit/test_bigquery_source.py +++ b/metadata-ingestion/tests/unit/test_bigquery_source.py @@ -13,6 +13,7 @@ from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.source.bigquery_v2.bigquery import BigqueryV2Source from datahub.ingestion.source.bigquery_v2.bigquery_audit import ( + _BIGQUERY_DEFAULT_SHARDED_TABLE_REGEX, BigqueryTableIdentifier, BigQueryTableRef, ) @@ -652,7 +653,7 @@ def test_get_table_and_shard_default( ) -> None: with patch( "datahub.ingestion.source.bigquery_v2.bigquery_audit.BigqueryTableIdentifier._BIGQUERY_DEFAULT_SHARDED_TABLE_REGEX", - "((.+)[_$])?(\\d{8})$", + _BIGQUERY_DEFAULT_SHARDED_TABLE_REGEX, ): assert BigqueryTableIdentifier.get_table_and_shard(table_name) == ( expected_table_prefix, From 12543bca3371fc33d1b3e3f6efc94afffbab2f72 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Mon, 10 Jul 2023 11:57:35 -0400 Subject: [PATCH 113/222] fix(config) Set search and browse flags default off (#8378) --- .../factories/src/main/resources/application.yml | 6 +++--- smoke-test/tests/cypress/cypress/e2e/browse/browseV2.js | 2 ++ smoke-test/tests/cypress/cypress/e2e/mutations/domains.js | 4 +--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/metadata-service/factories/src/main/resources/application.yml b/metadata-service/factories/src/main/resources/application.yml index d028c73c1420c..59f34e0b6e199 100644 --- a/metadata-service/factories/src/main/resources/application.yml +++ b/metadata-service/factories/src/main/resources/application.yml @@ -266,7 +266,7 @@ bootstrap: upgradeDefaultBrowsePaths: enabled: ${UPGRADE_DEFAULT_BROWSE_PATHS_ENABLED:false} # enable to run the upgrade to migrate legacy default browse paths to new ones backfillBrowsePathsV2: - enabled: ${BACKFILL_BROWSE_PATHS_V2:true} # Enables running the backfill of browsePathsV2 upgrade step. There are concerns about the load of this step so hiding it behind a flag. + enabled: ${BACKFILL_BROWSE_PATHS_V2:false} # Enables running the backfill of browsePathsV2 upgrade step. There are concerns about the load of this step so hiding it behind a flag. systemUpdate: initialBackOffMs: ${BOOTSTRAP_SYSTEM_UPDATE_INITIAL_BACK_OFF_MILLIS:5000} @@ -285,8 +285,8 @@ featureFlags: alwaysEmitChangeLog: ${ALWAYS_EMIT_CHANGE_LOG:false} # Enables always emitting a MCL even when no changes are detected. Used for Time Based Lineage when no changes occur. searchServiceDiffModeEnabled: ${SEARCH_SERVICE_DIFF_MODE_ENABLED:true} # Enables diff mode for search document writes, reduces amount of writes to ElasticSearch documents for no-ops readOnlyModeEnabled: ${READ_ONLY_MODE_ENABLED:false} # Enables read only mode for an instance. Right now this only affects ability to edit user profile image URL but can be extended - showSearchFiltersV2: ${SHOW_SEARCH_FILTERS_V2:true} # Enables showing the search filters V2 experience. - showBrowseV2: ${SHOW_BROWSE_V2:true} # Enables showing the browse v2 sidebar experience. + showSearchFiltersV2: ${SHOW_SEARCH_FILTERS_V2:false} # Enables showing the search filters V2 experience. + showBrowseV2: ${SHOW_BROWSE_V2:false} # Enables showing the browse v2 sidebar experience. preProcessHooks: uiEnabled: ${PRE_PROCESS_HOOKS_UI_ENABLED:true} # Circumvents Kafka for processing index updates for UI changes sourced from GraphQL to avoid processing delays showAcrylInfo: ${SHOW_ACRYL_INFO:false} # Show different CTAs within DataHub around moving to Managed DataHub. Set to true for the demo site. diff --git a/smoke-test/tests/cypress/cypress/e2e/browse/browseV2.js b/smoke-test/tests/cypress/cypress/e2e/browse/browseV2.js index 144531dcf2dae..a61b9030b13c6 100644 --- a/smoke-test/tests/cypress/cypress/e2e/browse/browseV2.js +++ b/smoke-test/tests/cypress/cypress/e2e/browse/browseV2.js @@ -13,6 +13,8 @@ describe("search", () => { req.reply((res) => { // Modify the response body directly res.body.data.appConfig.featureFlags.showBrowseV2 = isOn; + // search and browse both need to be on for browse to show + res.body.data.appConfig.featureFlags.showSearchFiltersV2 = isOn; }); } }); diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/domains.js b/smoke-test/tests/cypress/cypress/e2e/mutations/domains.js index 9114e2c843d98..39beb3bde161f 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutations/domains.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutations/domains.js @@ -34,10 +34,8 @@ describe("add remove domain", () => { it("search filter by domain", () => { cy.login(); cy.goToStarSearchList() - cy.get("[data-testid=filter-dropdown-Domain").click({ force: true }); cy.waitTextVisible(test_domain) - cy.get(`[data-testid="filter-option-${test_domain}"]`).click({ force: true }); - cy.get("[data-testid=update-filters]").click({ force: true }); + cy.get('[data-testid="facet-domains-' + test_domain_urn + '"]').click() cy.waitTextVisible("customers") }) From d92cbe99e03e22ae6cdf319eeef0c1f61b915630 Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Mon, 10 Jul 2023 12:27:29 -0400 Subject: [PATCH 114/222] test(ingest/kafka): Configure kafka connect tests for arm64 (#8362) --- .../kafka-connect/docker-compose.override.yml | 20 +- .../kafka-connect/setup/conf/mongo-init.js | 6 + .../kafka-connect/setup/conf/mongo-init.sh | 22 -- .../setup/conf/mongo-populate.js | 3 + .../kafka-connect/setup/connect.env | 3 +- .../kafka-connect/test_kafka_connect.py | 232 ++++++++---------- .../integration/kafka/docker-compose.yml | 6 +- 7 files changed, 128 insertions(+), 164 deletions(-) create mode 100644 metadata-ingestion/tests/integration/kafka-connect/setup/conf/mongo-init.js delete mode 100644 metadata-ingestion/tests/integration/kafka-connect/setup/conf/mongo-init.sh create mode 100644 metadata-ingestion/tests/integration/kafka-connect/setup/conf/mongo-populate.js diff --git a/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml b/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml index 3fd1af4f1d3e6..dbb4f84c0a655 100644 --- a/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml +++ b/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml @@ -1,9 +1,8 @@ --- version: '3.8' services: - connect: - image: confluentinc/cp-kafka-connect:6.2.1 + image: confluentinc/cp-kafka-connect:7.4.0 env_file: ./../kafka-connect/setup/connect.env container_name: test_connect hostname: test_connect @@ -35,7 +34,7 @@ services: # #confluent-hub install --no-prompt wepay/kafka-connect-bigquery:1.6.8 # - confluent-hub install --no-prompt mongodb/kafka-connect-mongodb:1.8.0 + confluent-hub install --no-prompt mongodb/kafka-connect-mongodb:1.10.1 # curl -k -SL "https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.27.tar.gz" \ | tar -xzf - -C /usr/share/confluent-hub-components/confluentinc-kafka-connect-jdbc/lib \ @@ -50,7 +49,7 @@ services: # sleep infinity mysqldb: - image: mysql:5.7 + image: mysql:8.0 environment: MYSQL_ROOT_PASSWORD: rootpwd MYSQL_USER: foo @@ -76,19 +75,14 @@ services: mongo: hostname: mongo - image: mongo:4.2.9 + image: mongo:6.0.7 container_name: "test_mongo" ports: - "27017:27017" command: --replSet rs0 environment: - - MONGO_INITDB_ROOT_USERNAME=admin - - MONGO_INITDB_ROOT_PASSWORD=admin + # Don't set mongo admin user or password to avoid KeyFile authentication + # which is required when running MongoDB as a replica set - MONGO_INITDB_DATABASE=test_db - - MONGO_INITDB_USERNAME=kafka-connector - - MONGO_INITDB_PASSWORD=password volumes: - - ./../kafka-connect/setup/conf/mongo-init.sh:/docker-entrypoint-initdb.d/mongo-init.sh:ro - -volumes: - test_zkdata: \ No newline at end of file + - ./../kafka-connect/setup/conf/:/scripts/ diff --git a/metadata-ingestion/tests/integration/kafka-connect/setup/conf/mongo-init.js b/metadata-ingestion/tests/integration/kafka-connect/setup/conf/mongo-init.js new file mode 100644 index 0000000000000..6c563f30fba2d --- /dev/null +++ b/metadata-ingestion/tests/integration/kafka-connect/setup/conf/mongo-init.js @@ -0,0 +1,6 @@ +rsconf = { + _id: "rs0", + members: [{ _id: 0, host: "test_mongo:27017", priority: 1.0 }], +}; +rs.initiate(rsconf); +rs.status(); diff --git a/metadata-ingestion/tests/integration/kafka-connect/setup/conf/mongo-init.sh b/metadata-ingestion/tests/integration/kafka-connect/setup/conf/mongo-init.sh deleted file mode 100644 index acd8424e5e7c2..0000000000000 --- a/metadata-ingestion/tests/integration/kafka-connect/setup/conf/mongo-init.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -mongo -- "$MONGO_INITDB_DATABASE" <<-EOJS - conn = new Mongo(); - db = conn.getDB("test_db"); - db.purchases.insertOne({ _id: 3, item: "lamp post", price: 12 }); - db.purchases.insertOne({ _id: 4, item: "lamp post", price: 13 }); -EOJS - - -{ -sleep 3 && -mongo -- "$MONGO_INITDB_DATABASE" <<-EOJS - var rootUser = '$MONGO_INITDB_ROOT_USERNAME'; - var rootPassword = '$MONGO_INITDB_ROOT_PASSWORD'; - var admin = db.getSiblingDB('admin'); - admin.auth(rootUser, rootPassword); -EOJS -} & - - - diff --git a/metadata-ingestion/tests/integration/kafka-connect/setup/conf/mongo-populate.js b/metadata-ingestion/tests/integration/kafka-connect/setup/conf/mongo-populate.js new file mode 100644 index 0000000000000..31dc59aec4afa --- /dev/null +++ b/metadata-ingestion/tests/integration/kafka-connect/setup/conf/mongo-populate.js @@ -0,0 +1,3 @@ +db.createCollection("purchases"); +db.purchases.insertOne({ _id: 3, item: "lamp post", price: 12 }); +db.purchases.insertOne({ _id: 4, item: "lamp post", price: 13 }); diff --git a/metadata-ingestion/tests/integration/kafka-connect/setup/connect.env b/metadata-ingestion/tests/integration/kafka-connect/setup/connect.env index ade5104605156..7b25ae37b10a6 100644 --- a/metadata-ingestion/tests/integration/kafka-connect/setup/connect.env +++ b/metadata-ingestion/tests/integration/kafka-connect/setup/connect.env @@ -1,5 +1,6 @@ CONNECT_BOOTSTRAP_SERVERS=test_broker:9092 CONNECT_REST_PORT=28083 +CONNECT_LISTENERS=http://:28083 CONNECT_GROUP_ID=kafka-connect CONNECT_CONFIG_STORAGE_TOPIC=_connect-configs CONNECT_OFFSET_STORAGE_TOPIC=_connect-offsets @@ -22,4 +23,4 @@ CONNECT_PLUGIN_PATH=/usr/share/confluent-hub-components, /usr/local/share/kafka/ CONNECT_CONFIG_PROVIDERS=env CONNECT_CONFIG_PROVIDERS_ENV_CLASS=io.strimzi.kafka.EnvVarConfigProvider MYSQL_CONNECTION_URL=jdbc:mysql://foo:datahub@test_mysql:3306/librarydb -POSTGRES_CONNECTION_URL=jdbc:postgresql://test_postgres:5432/postgres?user=postgres&password=datahub \ No newline at end of file +POSTGRES_CONNECTION_URL=jdbc:postgresql://test_postgres:5432/postgres?user=postgres&password=datahub diff --git a/metadata-ingestion/tests/integration/kafka-connect/test_kafka_connect.py b/metadata-ingestion/tests/integration/kafka-connect/test_kafka_connect.py index 8efea3ced6254..aaa39cd5f52d9 100644 --- a/metadata-ingestion/tests/integration/kafka-connect/test_kafka_connect.py +++ b/metadata-ingestion/tests/integration/kafka-connect/test_kafka_connect.py @@ -1,5 +1,4 @@ import subprocess -import time from typing import Any, Dict, List, cast from unittest import mock @@ -82,13 +81,8 @@ def test_resources_dir(pytestconfig): @pytest.fixture(scope="module") def loaded_kafka_connect(kafka_connect_runner): - # Set up the container. - time.sleep(10) - - # Setup mongo cluster - command = ( - 'docker exec test_mongo mongo admin -u admin -p admin --eval "rs.initiate();"' - ) + # # Setup mongo cluster + command = "docker exec test_mongo mongosh test_db -f /scripts/mongo-init.js" ret = subprocess.run( command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) @@ -99,17 +93,17 @@ def loaded_kafka_connect(kafka_connect_runner): KAFKA_CONNECT_ENDPOINT, headers={"Content-Type": "application/json"}, data="""{ - "name": "mysql_source1", - "config": { - "connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector", - "mode": "incrementing", - "incrementing.column.name": "id", - "topic.prefix": "test-mysql-jdbc-", - "tasks.max": "1", - "connection.url": "${env:MYSQL_CONNECTION_URL}" - } - } - """, + "name": "mysql_source1", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector", + "mode": "incrementing", + "incrementing.column.name": "id", + "topic.prefix": "test-mysql-jdbc-", + "tasks.max": "1", + "connection.url": "${env:MYSQL_CONNECTION_URL}" + } + } + """, ) assert r.status_code == 201 # Created # Creating MySQL source with regex router transformations , only topic prefix @@ -117,20 +111,20 @@ def loaded_kafka_connect(kafka_connect_runner): KAFKA_CONNECT_ENDPOINT, headers={"Content-Type": "application/json"}, data="""{ - "name": "mysql_source2", - "config": { - "connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector", - "mode": "incrementing", - "incrementing.column.name": "id", - "tasks.max": "1", - "connection.url": "${env:MYSQL_CONNECTION_URL}", - "transforms": "TotalReplacement", - "transforms.TotalReplacement.type": "org.apache.kafka.connect.transforms.RegexRouter", - "transforms.TotalReplacement.regex": ".*(book)", - "transforms.TotalReplacement.replacement": "my-new-topic-$1" - } - } - """, + "name": "mysql_source2", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector", + "mode": "incrementing", + "incrementing.column.name": "id", + "tasks.max": "1", + "connection.url": "${env:MYSQL_CONNECTION_URL}", + "transforms": "TotalReplacement", + "transforms.TotalReplacement.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.TotalReplacement.regex": ".*(book)", + "transforms.TotalReplacement.replacement": "my-new-topic-$1" + } + } + """, ) assert r.status_code == 201 # Created # Creating MySQL source with regex router transformations , no topic prefix, table whitelist @@ -138,21 +132,21 @@ def loaded_kafka_connect(kafka_connect_runner): KAFKA_CONNECT_ENDPOINT, headers={"Content-Type": "application/json"}, data="""{ - "name": "mysql_source3", - "config": { - "connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector", - "mode": "incrementing", - "incrementing.column.name": "id", - "table.whitelist": "book", - "tasks.max": "1", - "connection.url": "${env:MYSQL_CONNECTION_URL}", - "transforms": "TotalReplacement", - "transforms.TotalReplacement.type": "org.apache.kafka.connect.transforms.RegexRouter", - "transforms.TotalReplacement.regex": ".*", - "transforms.TotalReplacement.replacement": "my-new-topic" - } - } - """, + "name": "mysql_source3", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector", + "mode": "incrementing", + "incrementing.column.name": "id", + "table.whitelist": "book", + "tasks.max": "1", + "connection.url": "${env:MYSQL_CONNECTION_URL}", + "transforms": "TotalReplacement", + "transforms.TotalReplacement.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.TotalReplacement.regex": ".*", + "transforms.TotalReplacement.replacement": "my-new-topic" + } + } + """, ) assert r.status_code == 201 # Created # Creating MySQL source with query , topic prefix @@ -201,17 +195,17 @@ def loaded_kafka_connect(kafka_connect_runner): KAFKA_CONNECT_ENDPOINT, headers={"Content-Type": "application/json"}, data="""{ - "name": "mysql_sink", - "config": { - "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", - "insert.mode": "insert", - "auto.create": true, - "topics": "my-topic", - "tasks.max": "1", - "connection.url": "${env:MYSQL_CONNECTION_URL}" - } - } - """, + "name": "mysql_sink", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", + "insert.mode": "insert", + "auto.create": true, + "topics": "my-topic", + "tasks.max": "1", + "connection.url": "${env:MYSQL_CONNECTION_URL}" + } + } + """, ) assert r.status_code == 201 # Created @@ -220,21 +214,22 @@ def loaded_kafka_connect(kafka_connect_runner): KAFKA_CONNECT_ENDPOINT, headers={"Content-Type": "application/json"}, data="""{ - "name": "debezium-mysql-connector", - "config": { - "name": "debezium-mysql-connector", - "connector.class": "io.debezium.connector.mysql.MySqlConnector", - "database.hostname": "test_mysql", - "database.port": "3306", - "database.user": "root", - "database.password": "rootpwd", - "database.server.name": "debezium.topics", - "database.history.kafka.bootstrap.servers": "test_broker:9092", - "database.history.kafka.topic": "dbhistory.debeziummysql", - "include.schema.changes": "false" - } - } - """, + "name": "debezium-mysql-connector", + "config": { + "name": "debezium-mysql-connector", + "connector.class": "io.debezium.connector.mysql.MySqlConnector", + "database.hostname": "test_mysql", + "database.port": "3306", + "database.user": "root", + "database.password": "rootpwd", + "database.server.name": "debezium.topics", + "database.history.kafka.bootstrap.servers": "test_broker:9092", + "database.history.kafka.topic": "dbhistory.debeziummysql", + "database.allowPublicKeyRetrieval": "true", + "include.schema.changes": "false" + } + } + """, ) assert r.status_code == 201 # Created @@ -243,17 +238,17 @@ def loaded_kafka_connect(kafka_connect_runner): KAFKA_CONNECT_ENDPOINT, headers={"Content-Type": "application/json"}, data="""{ - "name": "postgres_source", - "config": { - "connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector", - "mode": "incrementing", - "incrementing.column.name": "id", - "table.whitelist": "member", - "topic.prefix": "test-postgres-jdbc-", - "tasks.max": "1", - "connection.url": "${env:POSTGRES_CONNECTION_URL}" - } - }""", + "name": "postgres_source", + "config": { + "connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector", + "mode": "incrementing", + "incrementing.column.name": "id", + "table.whitelist": "member", + "topic.prefix": "test-postgres-jdbc-", + "tasks.max": "1", + "connection.url": "${env:POSTGRES_CONNECTION_URL}" + } + }""", ) assert r.status_code == 201 # Created @@ -262,19 +257,19 @@ def loaded_kafka_connect(kafka_connect_runner): KAFKA_CONNECT_ENDPOINT, headers={"Content-Type": "application/json"}, data="""{ - "name": "generic_source", - "config": { - "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector", - "kafka.topic": "my-topic", - "quickstart": "product", - "key.converter": "org.apache.kafka.connect.storage.StringConverter", - "value.converter": "org.apache.kafka.connect.json.JsonConverter", - "value.converter.schemas.enable": "false", - "max.interval": 1000, - "iterations": 10000000, - "tasks.max": "1" - } - }""", + "name": "generic_source", + "config": { + "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector", + "kafka.topic": "my-topic", + "quickstart": "product", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "value.converter.schemas.enable": "false", + "max.interval": 1000, + "iterations": 10000000, + "tasks.max": "1" + } + }""", ) r.raise_for_status() assert r.status_code == 201 # Created @@ -284,37 +279,24 @@ def loaded_kafka_connect(kafka_connect_runner): KAFKA_CONNECT_ENDPOINT, headers={"Content-Type": "application/json"}, data=r"""{ - "name": "source_mongodb_connector", - "config": { - "tasks.max": "1", - "connector.class": "com.mongodb.kafka.connect.MongoSourceConnector", - "connection.uri": "mongodb://admin:admin@test_mongo:27017", - "topic.prefix": "mongodb", - "database": "test_db", - "collection": "purchases", - "copy.existing": true, - "copy.existing.namespace.regex": "test_db.purchases", - "change.stream.full.document": "updateLookup", - "topic.creation.enable": "true", - "topic.creation.default.replication.factor": "-1", - "topic.creation.default.partitions": "-1", - "output.json.formatter": "com.mongodb.kafka.connect.source.json.formatter.SimplifiedJson", - "key.converter": "org.apache.kafka.connect.storage.StringConverter", - "value.converter": "org.apache.kafka.connect.storage.StringConverter", - "key.converter.schemas.enable": false, - "value.converter.schemas.enable": false, - "output.format.key": "schema", - "output.format.value": "json", - "output.schema.infer.value": false, - "publish.full.document.only":true - } - }""", + "name": "source_mongodb_connector", + "config": { + "connector.class": "com.mongodb.kafka.connect.MongoSourceConnector", + "connection.uri": "mongodb://test_mongo:27017", + "topic.prefix": "mongodb", + "database": "test_db", + "collection": "purchases" + } + }""", ) r.raise_for_status() assert r.status_code == 201 # Created - # Give time for connectors to process the table data - time.sleep(60) + command = "docker exec test_mongo mongosh test_db -f /scripts/mongo-populate.js" + ret = subprocess.run( + command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + assert ret.returncode == 0 @freeze_time(FROZEN_TIME) diff --git a/metadata-ingestion/tests/integration/kafka/docker-compose.yml b/metadata-ingestion/tests/integration/kafka/docker-compose.yml index 5983b9ae704f3..43f30cbe1e665 100644 --- a/metadata-ingestion/tests/integration/kafka/docker-compose.yml +++ b/metadata-ingestion/tests/integration/kafka/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.8' services: zookeeper: - image: confluentinc/cp-zookeeper:7.2.0 + image: confluentinc/cp-zookeeper:7.2.2 env_file: zookeeper.env hostname: test_zookeeper container_name: test_zookeeper @@ -12,7 +12,7 @@ services: - test_zkdata:/var/opt/zookeeper broker: - image: confluentinc/cp-kafka:7.2.0 + image: confluentinc/cp-kafka:7.2.2 env_file: broker.env hostname: test_broker container_name: test_broker @@ -22,7 +22,7 @@ services: - "29092:29092" schema-registry: - image: confluentinc/cp-schema-registry:7.2.0 + image: confluentinc/cp-schema-registry:7.2.2 restart: unless-stopped env_file: schema-registry.env container_name: test_schema_registry From 64697eb5a0e1a9b91ce0ab43852881a2a9c24c80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20L=C3=BCdin?= <13187726+Masterchen09@users.noreply.github.com> Date: Mon, 10 Jul 2023 19:47:46 +0200 Subject: [PATCH 115/222] fix(ui): fix a too much recursion error when column lineage is highlighted (#8207) Co-authored-by: Chris Collins --- .../src/app/lineage/utils/highlightColumnLineage.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/datahub-web-react/src/app/lineage/utils/highlightColumnLineage.ts b/datahub-web-react/src/app/lineage/utils/highlightColumnLineage.ts index 3fe35b3605466..eaff8f958837b 100644 --- a/datahub-web-react/src/app/lineage/utils/highlightColumnLineage.ts +++ b/datahub-web-react/src/app/lineage/utils/highlightColumnLineage.ts @@ -1,5 +1,5 @@ +import { isEqual } from 'lodash'; import { ColumnEdge } from '../types'; -import { isSameColumn } from './columnLineageUtils'; function highlightDownstreamColumnLineage( sourceField: string, @@ -12,8 +12,9 @@ function highlightDownstreamColumnLineage( Object.entries(forwardLineage).forEach((entry) => { const [targetUrn, fieldPaths] = entry; (fieldPaths as string[]).forEach((targetField) => { - edges.push({ sourceUrn, sourceField, targetUrn, targetField }); - if (!isSameColumn({ sourceUrn, targetUrn, sourceField, targetField })) { + const edge: ColumnEdge = { sourceUrn, sourceField, targetUrn, targetField }; + if (!edges.some((value) => isEqual(value, edge))) { + edges.push(edge); highlightDownstreamColumnLineage(targetField, targetUrn, edges, fineGrainedMap); } }); @@ -32,8 +33,9 @@ function highlightUpstreamColumnLineage( Object.entries(reverseLineage).forEach((entry) => { const [sourceUrn, fieldPaths] = entry; (fieldPaths as string[]).forEach((sourceField) => { - edges.push({ targetUrn, targetField, sourceUrn, sourceField }); - if (!isSameColumn({ sourceUrn, targetUrn, sourceField, targetField })) { + const edge: ColumnEdge = { sourceUrn, sourceField, targetUrn, targetField }; + if (!edges.some((value) => isEqual(value, edge))) { + edges.push(edge); highlightUpstreamColumnLineage(sourceField, sourceUrn, edges, fineGrainedMap); } }); From e4736d7b46307f57e702bdc13289f40404167b92 Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Mon, 10 Jul 2023 20:40:06 +0200 Subject: [PATCH 116/222] fix(ingest/s3): Deequ import rearragement (#8389) --- .../src/datahub/ingestion/source/s3/config.py | 2 +- .../source/s3/datalake_profiler_config.py | 87 +++++++++++++++++++ .../datahub/ingestion/source/s3/profiling.py | 86 +----------------- .../src/datahub/ingestion/source/s3/source.py | 13 ++- .../mysql/mysql_table_level_only.json | 55 ++++++++++++ 5 files changed, 155 insertions(+), 88 deletions(-) create mode 100644 metadata-ingestion/src/datahub/ingestion/source/s3/datalake_profiler_config.py diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/config.py b/metadata-ingestion/src/datahub/ingestion/source/s3/config.py index 97e2f52077c41..c7559a222d51d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/s3/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/s3/config.py @@ -11,7 +11,7 @@ from datahub.ingestion.source.aws.aws_common import AwsConnectionConfig from datahub.ingestion.source.data_lake_common.config import PathSpecsConfigMixin from datahub.ingestion.source.data_lake_common.path_spec import PathSpec -from datahub.ingestion.source.s3.profiling import DataLakeProfilerConfig +from datahub.ingestion.source.s3.datalake_profiler_config import DataLakeProfilerConfig from datahub.ingestion.source.state.stale_entity_removal_handler import ( StatefulStaleMetadataRemovalConfig, ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/datalake_profiler_config.py b/metadata-ingestion/src/datahub/ingestion/source/s3/datalake_profiler_config.py new file mode 100644 index 0000000000000..acbe46de8583a --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/s3/datalake_profiler_config.py @@ -0,0 +1,87 @@ +from typing import Any, Dict, Optional + +import pydantic +from pydantic.fields import Field + +from datahub.configuration import ConfigModel +from datahub.configuration.common import AllowDenyPattern + + +class DataLakeProfilerConfig(ConfigModel): + enabled: bool = Field( + default=False, description="Whether profiling should be done." + ) + + # These settings will override the ones below. + profile_table_level_only: bool = Field( + default=False, + description="Whether to perform profiling at table-level only or include column-level profiling as well.", + ) + + _allow_deny_patterns: AllowDenyPattern = pydantic.PrivateAttr( + default=AllowDenyPattern.allow_all(), + ) + + max_number_of_fields_to_profile: Optional[pydantic.PositiveInt] = Field( + default=None, + description="A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.", + ) + + include_field_null_count: bool = Field( + default=True, + description="Whether to profile for the number of nulls for each column.", + ) + include_field_min_value: bool = Field( + default=True, + description="Whether to profile for the min value of numeric columns.", + ) + include_field_max_value: bool = Field( + default=True, + description="Whether to profile for the max value of numeric columns.", + ) + include_field_mean_value: bool = Field( + default=True, + description="Whether to profile for the mean value of numeric columns.", + ) + include_field_median_value: bool = Field( + default=True, + description="Whether to profile for the median value of numeric columns.", + ) + include_field_stddev_value: bool = Field( + default=True, + description="Whether to profile for the standard deviation of numeric columns.", + ) + include_field_quantiles: bool = Field( + default=True, + description="Whether to profile for the quantiles of numeric columns.", + ) + include_field_distinct_value_frequencies: bool = Field( + default=True, description="Whether to profile for distinct value frequencies." + ) + include_field_histogram: bool = Field( + default=True, + description="Whether to profile for the histogram for numeric fields.", + ) + include_field_sample_values: bool = Field( + default=True, + description="Whether to profile for the sample values for all columns.", + ) + + @pydantic.root_validator() + def ensure_field_level_settings_are_normalized( + cls: "DataLakeProfilerConfig", values: Dict[str, Any] + ) -> Dict[str, Any]: + max_num_fields_to_profile_key = "max_number_of_fields_to_profile" + max_num_fields_to_profile = values.get(max_num_fields_to_profile_key) + + # Disable all field-level metrics. + if values.get("profile_table_level_only"): + for field_level_metric in cls.__fields__: + if field_level_metric.startswith("include_field_"): + values.setdefault(field_level_metric, False) + + assert ( + max_num_fields_to_profile is None + ), f"{max_num_fields_to_profile_key} should be set to None" + + return values diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/profiling.py b/metadata-ingestion/src/datahub/ingestion/source/s3/profiling.py index 3e30449b5a62e..c969b229989e8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/s3/profiling.py +++ b/metadata-ingestion/src/datahub/ingestion/source/s3/profiling.py @@ -1,9 +1,7 @@ import dataclasses -from typing import Any, Dict, List, Optional +from typing import Any, List, Optional -import pydantic from pandas import DataFrame -from pydantic.fields import Field from pydeequ.analyzers import ( AnalysisRunBuilder, AnalysisRunner, @@ -33,12 +31,12 @@ TimestampType, ) -from datahub.configuration.common import AllowDenyPattern, ConfigModel from datahub.emitter.mce_builder import get_sys_time from datahub.ingestion.source.profiling.common import ( Cardinality, convert_to_cardinality, ) +from datahub.ingestion.source.s3.datalake_profiler_config import DataLakeProfilerConfig from datahub.ingestion.source.s3.report import DataLakeSourceReport from datahub.metadata.schema_classes import ( DatasetFieldProfileClass, @@ -59,86 +57,6 @@ def null_str(value: Any) -> Optional[str]: return str(value) if value is not None else None -class DataLakeProfilerConfig(ConfigModel): - enabled: bool = Field( - default=False, description="Whether profiling should be done." - ) - - # These settings will override the ones below. - profile_table_level_only: bool = Field( - default=False, - description="Whether to perform profiling at table-level only or include column-level profiling as well.", - ) - - _allow_deny_patterns: AllowDenyPattern = pydantic.PrivateAttr( - default=AllowDenyPattern.allow_all(), - ) - - max_number_of_fields_to_profile: Optional[pydantic.PositiveInt] = Field( - default=None, - description="A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.", - ) - - include_field_null_count: bool = Field( - default=True, - description="Whether to profile for the number of nulls for each column.", - ) - include_field_min_value: bool = Field( - default=True, - description="Whether to profile for the min value of numeric columns.", - ) - include_field_max_value: bool = Field( - default=True, - description="Whether to profile for the max value of numeric columns.", - ) - include_field_mean_value: bool = Field( - default=True, - description="Whether to profile for the mean value of numeric columns.", - ) - include_field_median_value: bool = Field( - default=True, - description="Whether to profile for the median value of numeric columns.", - ) - include_field_stddev_value: bool = Field( - default=True, - description="Whether to profile for the standard deviation of numeric columns.", - ) - include_field_quantiles: bool = Field( - default=True, - description="Whether to profile for the quantiles of numeric columns.", - ) - include_field_distinct_value_frequencies: bool = Field( - default=True, description="Whether to profile for distinct value frequencies." - ) - include_field_histogram: bool = Field( - default=True, - description="Whether to profile for the histogram for numeric fields.", - ) - include_field_sample_values: bool = Field( - default=True, - description="Whether to profile for the sample values for all columns.", - ) - - @pydantic.root_validator() - def ensure_field_level_settings_are_normalized( - cls: "DataLakeProfilerConfig", values: Dict[str, Any] - ) -> Dict[str, Any]: - max_num_fields_to_profile_key = "max_number_of_fields_to_profile" - max_num_fields_to_profile = values.get(max_num_fields_to_profile_key) - - # Disable all field-level metrics. - if values.get("profile_table_level_only"): - for field_level_metric in cls.__fields__: - if field_level_metric.startswith("include_field_"): - values.setdefault(field_level_metric, False) - - assert ( - max_num_fields_to_profile is None - ), f"{max_num_fields_to_profile_key} should be set to None" - - return values - - @dataclasses.dataclass class _SingleColumnSpec: column: str diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py index 1daaddd0b21a7..db83c29370987 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py @@ -9,9 +9,7 @@ from datetime import datetime from typing import Any, Dict, Iterable, List, Optional, Tuple -import pydeequ from more_itertools import peekable -from pydeequ.analyzers import AnalyzerContext from pyspark.conf import SparkConf from pyspark.sql import SparkSession from pyspark.sql.dataframe import DataFrame @@ -62,7 +60,6 @@ ) from datahub.ingestion.source.data_lake_common.data_lake_utils import ContainerWUCreator from datahub.ingestion.source.s3.config import DataLakeSourceConfig, PathSpec -from datahub.ingestion.source.s3.profiling import _SingleTableProfiler from datahub.ingestion.source.s3.report import DataLakeSourceReport from datahub.ingestion.source.schema_inference import avro, csv_tsv, json, parquet from datahub.ingestion.source.state.stale_entity_removal_handler import ( @@ -285,6 +282,10 @@ def __init__(self, config: DataLakeSourceConfig, ctx: PipelineContext): self.init_spark() def init_spark(self): + # Importing here to avoid Deequ dependency for non profiling use cases + # Deequ fails if Spark is not available which is not needed for non profiling use cases + import pydeequ + conf = SparkConf() conf.set( @@ -477,6 +478,12 @@ def get_fields(self, table_data: TableData, path_spec: PathSpec) -> List: def get_table_profile( self, table_data: TableData, dataset_urn: str ) -> Iterable[MetadataWorkUnit]: + # Importing here to avoid Deequ dependency for non profiling use cases + # Deequ fails if Spark is not available which is not needed for non profiling use cases + from pydeequ.analyzers import AnalyzerContext + + from datahub.ingestion.source.s3.profiling import _SingleTableProfiler + # read in the whole table with Spark for profiling table = None try: diff --git a/metadata-ingestion/tests/integration/mysql/mysql_table_level_only.json b/metadata-ingestion/tests/integration/mysql/mysql_table_level_only.json index 8b266a91bad06..cbe5f4e9e314d 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_table_level_only.json +++ b/metadata-ingestion/tests/integration/mysql/mysql_table_level_only.json @@ -66,6 +66,21 @@ "runId": "mysql-2020_04_14-07_00_00" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", @@ -218,6 +233,26 @@ "runId": "mysql-2020_04_14-07_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", + "urn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", @@ -346,6 +381,26 @@ "runId": "mysql-2020_04_14-07_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", + "urn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", From d8888cf0b7155bc86d12b0241b65e62bde5c8b97 Mon Sep 17 00:00:00 2001 From: Segun Sogunle <102164363+segun-s@users.noreply.github.com> Date: Mon, 10 Jul 2023 19:51:48 +0100 Subject: [PATCH 117/222] feat(ingest): Add disable flag for TopicRecordNameStrategy (#8224) Co-authored-by: Harshal Sheth --- .../ingestion/source/confluent_schema_registry.py | 15 ++++++++++++++- .../src/datahub/ingestion/source/kafka.py | 4 ++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py b/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py index 26b5410c78e15..dcaec4e45737f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py +++ b/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py @@ -80,8 +80,21 @@ def _get_subject_for_topic(self, topic: str, is_key_schema: bool) -> Optional[st # Subject name format when the schema registry subject name strategy is # (a) TopicNameStrategy(default strategy): - # (b) TopicRecordNameStrategy: -- + # there's a third case + # (c) TopicNameStrategy differing by environment name suffixes. + # e.g "a.b.c.d-value" and "a.b.c.d.qa-value" + # For such instances, the wrong schema registry entries could picked by the previous logic. for subject in self.known_schema_registry_subjects: - if subject.startswith(topic) and subject.endswith(subject_key_suffix): + if ( + self.source_config.disable_topic_record_naming_strategy + and subject == subject_key + ): + return subject + if ( + (not self.source_config.disable_topic_record_naming_strategy) + and subject.startswith(topic) + and subject.endswith(subject_key_suffix) + ): return subject return None diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka.py b/metadata-ingestion/src/datahub/ingestion/source/kafka.py index 6a0f8c4f2c89f..61f6103347eb3 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/kafka.py +++ b/metadata-ingestion/src/datahub/ingestion/source/kafka.py @@ -93,6 +93,10 @@ class KafkaSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin): default=False, description="Disables warnings reported for non-AVRO/Protobuf value or key schemas if set.", ) + disable_topic_record_naming_strategy: bool = pydantic.Field( + default=False, + description="Disables the utilization of the TopicRecordNameStrategy for Schema Registry subjects. For more information, visit: https://docs.confluent.io/platform/current/schema-registry/serdes-develop/index.html#handling-differences-between-preregistered-and-client-derived-schemas:~:text=io.confluent.kafka.serializers.subject.TopicRecordNameStrategy", + ) @dataclass From b97e9b5b53c25ea3a1306af7e908d4af643c3543 Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Mon, 10 Jul 2023 23:53:14 -0700 Subject: [PATCH 118/222] refactor(graphql): make graphql engine extensible (#8394) --- .../datahub/graphql/GmsGraphQLEngine.java | 51 +++++++++++++++++-- .../datahub/graphql/GmsGraphQLPlugin.java | 44 ++++++++++++++++ 2 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLPlugin.java diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 198c9d86c2117..378362a77654c 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -330,6 +330,7 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -426,6 +427,11 @@ public class GmsGraphQLEngine { private final DataProductType dataProductType; private final OwnershipType ownershipType; + /** + * A list of GraphQL Plugins that extend the core engine + */ + private final List graphQLPlugins; + /** * Configures the graph objects that can be fetched primary key. */ @@ -453,6 +459,12 @@ public class GmsGraphQLEngine { public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { + this.graphQLPlugins = List.of( + // Add new plugins here + ); + + this.graphQLPlugins.forEach(plugin -> plugin.init(args)); + this.entityClient = args.entityClient; this.graphClient = args.graphClient; this.usageClient = args.usageClient; @@ -558,6 +570,14 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { ownershipType ); this.loadableTypes = new ArrayList<>(entityTypes); + // Extend loadable types with types from the plugins + // This allows us to offer search and browse capabilities out of the box for those types + for (GmsGraphQLPlugin plugin: this.graphQLPlugins) { + Collection> pluginLoadableTypes = plugin.getLoadableTypes(); + if (pluginLoadableTypes != null) { + this.loadableTypes.addAll(pluginLoadableTypes); + } + } this.ownerTypes = ImmutableList.of(corpUserType, corpGroupType); this.searchableTypes = loadableTypes.stream() .filter(type -> (type instanceof SearchableEntityType)) @@ -573,7 +593,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { * Returns a {@link Supplier} responsible for creating a new {@link DataLoader} from * a {@link LoadableType}. */ - public Map>> loaderSuppliers(final List> loadableTypes) { + public Map>> loaderSuppliers(final Collection> loadableTypes) { return loadableTypes .stream() .collect(Collectors.toMap( @@ -582,6 +602,15 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { )); } + /** + * Final call to wire up any extra resolvers the plugin might want to add on + * @param builder + */ + private void configurePluginResolvers(final RuntimeWiring.Builder builder) { + this.graphQLPlugins.forEach(plugin -> plugin.configureExtraResolvers(builder)); + } + + public void configureRuntimeWiring(final RuntimeWiring.Builder builder) { configureQueryResolvers(builder); configureMutationResolvers(builder); @@ -619,10 +648,13 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) { configureViewResolvers(builder); configureQueryEntityResolvers(builder); configureOwnershipTypeResolver(builder); + configurePluginResolvers(builder); } + public GraphQLEngine.Builder builder() { - return GraphQLEngine.builder() + final GraphQLEngine.Builder builder = GraphQLEngine.builder(); + builder .addSchema(fileBasedSchema(GMS_SCHEMA_FILE)) .addSchema(fileBasedSchema(SEARCH_SCHEMA_FILE)) .addSchema(fileBasedSchema(APP_SCHEMA_FILE)) @@ -633,10 +665,23 @@ public GraphQLEngine.Builder builder() { .addSchema(fileBasedSchema(TIMELINE_SCHEMA_FILE)) .addSchema(fileBasedSchema(TESTS_SCHEMA_FILE)) .addSchema(fileBasedSchema(STEPS_SCHEMA_FILE)) - .addSchema(fileBasedSchema(LINEAGE_SCHEMA_FILE)) + .addSchema(fileBasedSchema(LINEAGE_SCHEMA_FILE)); + + for (GmsGraphQLPlugin plugin: this.graphQLPlugins) { + List pluginSchemaFiles = plugin.getSchemaFiles(); + if (pluginSchemaFiles != null) { + pluginSchemaFiles.forEach(schema -> builder.addSchema(fileBasedSchema(schema))); + } + Collection> pluginLoadableTypes = plugin.getLoadableTypes(); + if (pluginLoadableTypes != null) { + pluginLoadableTypes.forEach(loadableType -> builder.addDataLoaders(loaderSuppliers(pluginLoadableTypes))); + } + } + builder .addDataLoaders(loaderSuppliers(loadableTypes)) .addDataLoader("Aspect", context -> createDataLoader(aspectType, context)) .configureRuntimeWiring(this::configureRuntimeWiring); + return builder; } public static String fileBasedSchema(String fileName) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLPlugin.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLPlugin.java new file mode 100644 index 0000000000000..ae1e26b610a61 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLPlugin.java @@ -0,0 +1,44 @@ +package com.linkedin.datahub.graphql; + +import com.linkedin.datahub.graphql.types.LoadableType; +import graphql.schema.idl.RuntimeWiring; +import java.util.Collection; +import java.util.List; + + +/** + * An interface that allows the Core GMS GraphQL Engine to be extended without requiring + * code changes in the GmsGraphQLEngine class if new entities, relationships or resolvers + * need to be introduced. This is useful if you are maintaining a fork of DataHub and + * don't want to deal with merge conflicts. + */ +public interface GmsGraphQLPlugin { + + /** + * Initialization method that allows the plugin to instantiate + * @param args + */ + void init(GmsGraphQLEngineArgs args); + + /** + * Return a list of schema files that contain graphql definitions + * that are served by this plugin + * @return + */ + List getSchemaFiles(); + + /** + * Return a list of LoadableTypes that this plugin serves + * @return + */ + Collection> getLoadableTypes(); + + /** + * Optional callback that a plugin can implement to configure any Query, Mutation or Type specific resolvers. + * @param wiringBuilder + */ + default void configureExtraResolvers(final RuntimeWiring.Builder wiringBuilder) { + + } + +} From 18c1f124361fe05760f393c03051a9acabcc1b4c Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Tue, 11 Jul 2023 10:30:47 -0400 Subject: [PATCH 119/222] feat(ui) Allow a configurable default tab for domain entity profile page (#8316) --- .../resolvers/config/AppConfigResolver.java | 11 ++++++++ .../src/main/resources/app.graphql | 27 +++++++++++++++++++ .../src/app/entity/domain/DomainEntity.tsx | 4 +++ .../src/app/entity/shared/constants.ts | 7 +++++ .../containers/profile/EntityProfile.tsx | 6 ++++- .../entity/shared/containers/profile/utils.ts | 21 ++++++++++++++- .../src/app/entity/shared/types.ts | 1 + datahub-web-react/src/appConfigContext.tsx | 3 +++ datahub-web-react/src/graphql/app.graphql | 5 ++++ .../metadata/config/EntityProfileConfig.java | 12 +++++++++ .../metadata/config/VisualConfiguration.java | 5 ++++ .../src/main/resources/application.yml | 3 +++ 12 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 metadata-io/src/main/java/com/linkedin/metadata/config/EntityProfileConfig.java diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java index 026ac719f04ea..876f3f08c8e25 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java @@ -7,6 +7,8 @@ import com.linkedin.datahub.graphql.generated.AnalyticsConfig; import com.linkedin.datahub.graphql.generated.AppConfig; import com.linkedin.datahub.graphql.generated.AuthConfig; +import com.linkedin.datahub.graphql.generated.EntityProfileConfig; +import com.linkedin.datahub.graphql.generated.EntityProfilesConfig; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FeatureFlagsConfig; import com.linkedin.datahub.graphql.generated.IdentityManagementConfig; @@ -133,6 +135,15 @@ public CompletableFuture get(final DataFetchingEnvironment environmen queriesTabConfig.setQueriesTabResultSize(_visualConfiguration.getQueriesTab().getQueriesTabResultSize()); visualConfig.setQueriesTab(queriesTabConfig); } + if (_visualConfiguration != null && _visualConfiguration.getEntityProfile() != null) { + EntityProfilesConfig entityProfilesConfig = new EntityProfilesConfig(); + if (_visualConfiguration.getEntityProfile().getDomainDefaultTab() != null) { + EntityProfileConfig profileConfig = new EntityProfileConfig(); + profileConfig.setDefaultTab(_visualConfiguration.getEntityProfile().getDomainDefaultTab()); + entityProfilesConfig.setDomain(profileConfig); + } + visualConfig.setEntityProfiles(entityProfilesConfig); + } appConfig.setVisualConfig(visualConfig); final TelemetryConfig telemetryConfig = new TelemetryConfig(); diff --git a/datahub-graphql-core/src/main/resources/app.graphql b/datahub-graphql-core/src/main/resources/app.graphql index a761dc8de595c..37183bac13f0e 100644 --- a/datahub-graphql-core/src/main/resources/app.graphql +++ b/datahub-graphql-core/src/main/resources/app.graphql @@ -211,6 +211,11 @@ type VisualConfig { Configuration for the queries tab """ queriesTab: QueriesTabConfig + + """ + Configuration for the queries tab + """ + entityProfiles: EntityProfilesConfig } """ @@ -223,6 +228,28 @@ type QueriesTabConfig { queriesTabResultSize: Int } + +""" +Configuration for different entity profiles +""" +type EntityProfilesConfig { + """ + The configurations for a Domain entity profile + """ + domain: EntityProfileConfig +} + +""" +Configuration for an entity profile +""" +type EntityProfileConfig { + """ + The enum value from EntityProfileTab for which tab should be showed by default on + entity profile pages. If null, rely on default sorting from React code. + """ + defaultTab: String +} + """ Configurations related to tracking users in the app """ diff --git a/datahub-web-react/src/app/entity/domain/DomainEntity.tsx b/datahub-web-react/src/app/entity/domain/DomainEntity.tsx index e812ee87e6887..3b3045abe2a7c 100644 --- a/datahub-web-react/src/app/entity/domain/DomainEntity.tsx +++ b/datahub-web-react/src/app/entity/domain/DomainEntity.tsx @@ -13,6 +13,7 @@ import { DomainEntitiesTab } from './DomainEntitiesTab'; import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown'; import { EntityActionItem } from '../shared/entity/EntityActions'; import DataProductsTab from './DataProductsTab/DataProductsTab'; +import { EntityProfileTab } from '../shared/constants'; // import { EntityActionItem } from '../shared/entity/EntityActions'; /** @@ -72,14 +73,17 @@ export class DomainEntity implements Entity { isNameEditable tabs={[ { + id: EntityProfileTab.DOMAIN_ENTITIES_TAB, name: 'Entities', component: DomainEntitiesTab, }, { + id: EntityProfileTab.DOCUMENTATION_TAB, name: 'Documentation', component: DocumentationTab, }, { + id: EntityProfileTab.DATA_PRODUCTS_TAB, name: 'Data Products', component: DataProductsTab, }, diff --git a/datahub-web-react/src/app/entity/shared/constants.ts b/datahub-web-react/src/app/entity/shared/constants.ts index ba399dd287803..bc40ba871e7d1 100644 --- a/datahub-web-react/src/app/entity/shared/constants.ts +++ b/datahub-web-react/src/app/entity/shared/constants.ts @@ -93,3 +93,10 @@ export const GLOSSARY_ENTITY_TYPES = [EntityType.GlossaryTerm, EntityType.Glossa export const DEFAULT_SYSTEM_ACTOR_URNS = ['urn:li:corpuser:__datahub_system', 'urn:li:corpuser:unknown']; export const VIEW_ENTITY_PAGE = 'VIEW_ENTITY_PAGE'; + +// only values for Domain Entity for custom configurable default tab +export enum EntityProfileTab { + DOMAIN_ENTITIES_TAB = 'DOMAIN_ENTITIES_TAB', + DOCUMENTATION_TAB = 'DOCUMENTATION_TAB', + DATA_PRODUCTS_TAB = 'DATA_PRODUCTS_TAB', +} diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx index f6e839d727c02..8a559013c892c 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx @@ -8,6 +8,7 @@ import { Message } from '../../../../shared/Message'; import { getEntityPath, getOnboardingStepIdsForEntityType, + sortEntityProfileTabs, useRoutedTab, useUpdateGlossaryEntityDataOnChange, } from './utils'; @@ -43,6 +44,7 @@ import { LINEAGE_GRAPH_INTRO_ID, LINEAGE_GRAPH_TIME_FILTER_ID, } from '../../../../onboarding/config/LineageGraphOnboardingConfig'; +import { useAppConfig } from '../../../../useAppConfig'; type Props = { urn: string; @@ -168,8 +170,10 @@ export const EntityProfile = ({ const isHideSiblingMode = useIsSeparateSiblingsMode(); const entityRegistry = useEntityRegistry(); const history = useHistory(); + const appConfig = useAppConfig(); const isCompact = React.useContext(CompactContext); const tabsWithDefaults = tabs.map((tab) => ({ ...tab, display: { ...defaultTabDisplayConfig, ...tab.display } })); + const sortedTabs = sortEntityProfileTabs(appConfig.config, entityType, tabsWithDefaults); const sideBarSectionsWithDefaults = sidebarSections.map((sidebarSection) => ({ ...sidebarSection, display: { ...defaultSidebarSection, ...sidebarSection.display }, @@ -235,7 +239,7 @@ export const EntityProfile = ({ }, })) || []; - const visibleTabs = [...tabsWithDefaults, ...autoRenderTabs].filter((tab) => + const visibleTabs = [...sortedTabs, ...autoRenderTabs].filter((tab) => tab.display?.visible(entityData, dataPossiblyCombinedWithSiblings), ); diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/utils.ts b/datahub-web-react/src/app/entity/shared/containers/profile/utils.ts index 625272aec25b9..fabf0b2c51e95 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/utils.ts +++ b/datahub-web-react/src/app/entity/shared/containers/profile/utils.ts @@ -2,7 +2,7 @@ import { useEffect } from 'react'; import { useLocation } from 'react-router'; import queryString from 'query-string'; import { isEqual } from 'lodash'; -import { EntityType } from '../../../../../types.generated'; +import { AppConfig, EntityType } from '../../../../../types.generated'; import useIsLineageMode from '../../../../lineage/utils/useIsLineageMode'; import { useEntityRegistry } from '../../../../useEntityRegistry'; import EntityRegistry from '../../../EntityRegistry'; @@ -202,3 +202,22 @@ export function getOnboardingStepIdsForEntityType(entityType: EntityType): strin return []; } } + +function sortTabsWithDefaultTabId(tabs: EntityTab[], defaultTabId: string) { + return tabs.sort((tabA, tabB) => { + if (tabA.id === defaultTabId) return -1; + if (tabB.id === defaultTabId) return 1; + return 0; + }); +} + +export function sortEntityProfileTabs(appConfig: AppConfig, entityType: EntityType, tabs: EntityTab[]) { + const sortedTabs = [...tabs]; + + if (entityType === EntityType.Domain && appConfig.visualConfig.entityProfiles?.domain?.defaultTab) { + const defaultTabId = appConfig.visualConfig.entityProfiles?.domain.defaultTab; + sortTabsWithDefaultTabId(sortedTabs, defaultTabId); + } + + return sortedTabs; +} diff --git a/datahub-web-react/src/app/entity/shared/types.ts b/datahub-web-react/src/app/entity/shared/types.ts index 9b7c794840e4b..e36f5050a24b7 100644 --- a/datahub-web-react/src/app/entity/shared/types.ts +++ b/datahub-web-react/src/app/entity/shared/types.ts @@ -48,6 +48,7 @@ export type EntityTab = { enabled: (GenericEntityProperties, T) => boolean; // Whether the tab is enabled on the UI. Defaults to true. }; properties?: any; + id?: string; }; export type EntitySidebarSection = { diff --git a/datahub-web-react/src/appConfigContext.tsx b/datahub-web-react/src/appConfigContext.tsx index 6866278838b1e..3b34b108ecc93 100644 --- a/datahub-web-react/src/appConfigContext.tsx +++ b/datahub-web-react/src/appConfigContext.tsx @@ -24,6 +24,9 @@ export const DEFAULT_APP_CONFIG = { queriesTab: { queriesTabResultSize: 5, }, + entityProfile: { + domainDefaultTab: null, + }, }, authConfig: { tokenAuthEnabled: false, diff --git a/datahub-web-react/src/graphql/app.graphql b/datahub-web-react/src/graphql/app.graphql index f62822047fc32..4b1295f1024a2 100644 --- a/datahub-web-react/src/graphql/app.graphql +++ b/datahub-web-react/src/graphql/app.graphql @@ -40,6 +40,11 @@ query appConfig { queriesTab { queriesTabResultSize } + entityProfiles { + domain { + defaultTab + } + } } telemetryConfig { enableThirdPartyLogging diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/EntityProfileConfig.java b/metadata-io/src/main/java/com/linkedin/metadata/config/EntityProfileConfig.java new file mode 100644 index 0000000000000..7c4394d07bf9c --- /dev/null +++ b/metadata-io/src/main/java/com/linkedin/metadata/config/EntityProfileConfig.java @@ -0,0 +1,12 @@ +package com.linkedin.metadata.config; + +import lombok.Data; + + +@Data +public class EntityProfileConfig { + /** + * The default tab to show first on a Domain entity profile. Defaults to React code sorting if not present. + */ + public String domainDefaultTab; +} diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/VisualConfiguration.java b/metadata-io/src/main/java/com/linkedin/metadata/config/VisualConfiguration.java index 6505fdf29da16..d1c357186e1ae 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/config/VisualConfiguration.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/config/VisualConfiguration.java @@ -17,4 +17,9 @@ public class VisualConfiguration { * Queries tab related configurations */ public QueriesTabConfig queriesTab; + + /** + * Queries tab related configurations + */ + public EntityProfileConfig entityProfile; } diff --git a/metadata-service/factories/src/main/resources/application.yml b/metadata-service/factories/src/main/resources/application.yml index 59f34e0b6e199..18f042b65d0b1 100644 --- a/metadata-service/factories/src/main/resources/application.yml +++ b/metadata-service/factories/src/main/resources/application.yml @@ -108,6 +108,9 @@ visualConfig: assets: logoUrl: ${REACT_APP_LOGO_URL:/assets/platforms/datahublogo.png} faviconUrl: ${REACT_APP_FAVICON_URL:/assets/favicon.ico} + entityProfile: + # we only support default tab for domains right now. In order to implement for other entities, update React code + domainDefaultTab: ${DOMAIN_DEFAULT_TAB:} # set to DOCUMENTATION_TAB to show documentation tab first # Storage Layer From 2261531e3103953189d19d5a446d3d77df85dcdf Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Tue, 11 Jul 2023 10:39:47 -0400 Subject: [PATCH 120/222] test(ingest): Aspect level golden file comparison (#8310) --- metadata-ingestion/setup.py | 11 +- .../src/datahub/cli/check_cli.py | 47 +++- .../src/datahub/emitter/aspect.py | 3 + metadata-ingestion/src/datahub/emitter/mcp.py | 12 +- .../src/datahub/emitter/mcp_patch_builder.py | 3 +- .../datahub_ingestion_run_summary_provider.py | 3 +- .../src/datahub/ingestion/sink/file.py | 15 +- .../datahub/testing/compare_metadata_json.py | 109 +++++++ .../src/datahub/testing/mcp_diff.py | 266 ++++++++++++++++++ .../tests/test_helpers/mce_helpers.py | 121 ++------ .../unit/patch/complex_dataset_patch.json | 121 +++++--- .../tests/unit/test_bigquery_usage.py | 4 +- ...ce_helpers.py => test_compare_metadata.py} | 7 +- .../unit/test_generic_aspect_transformer.py | 7 +- .../tests/unit/test_workunit.py | 11 +- 15 files changed, 576 insertions(+), 164 deletions(-) create mode 100644 metadata-ingestion/src/datahub/testing/compare_metadata_json.py create mode 100644 metadata-ingestion/src/datahub/testing/mcp_diff.py rename metadata-ingestion/tests/unit/{test_mce_helpers.py => test_compare_metadata.py} (96%) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 87cd2bbbd1dba..47a9d7f88e6ae 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -424,6 +424,11 @@ def get_long_description(): "types-protobuf>=4.21.0.1", } + +pytest_dep = "pytest>=6.2.2" +deepdiff_dep = "deepdiff" +test_api_requirements = {pytest_dep, deepdiff_dep, "PyYAML"} + base_dev_requirements = { *base_requirements, *framework_common, @@ -442,11 +447,12 @@ def get_long_description(): # pydantic 1.8.2 is incompatible with mypy 0.910. # See https://github.com/samuelcolvin/pydantic/pull/3175#issuecomment-995382910. "pydantic>=1.9.0", - "pytest>=6.2.2", + *test_api_requirements, + pytest_dep, "pytest-asyncio>=0.16.0", "pytest-cov>=2.8.1", "pytest-docker>=1.0.1", - "deepdiff", + deepdiff_dep, "requests-mock", "freezegun", "jsonpickle", @@ -697,6 +703,7 @@ def get_long_description(): ) ), "dev": list(dev_requirements), + "testing-utils": list(test_api_requirements), # To import `datahub.testing` "integration-tests": list(full_test_dev_requirements), }, ) diff --git a/metadata-ingestion/src/datahub/cli/check_cli.py b/metadata-ingestion/src/datahub/cli/check_cli.py index 5d34967d496cd..5cee21e45b97a 100644 --- a/metadata-ingestion/src/datahub/cli/check_cli.py +++ b/metadata-ingestion/src/datahub/cli/check_cli.py @@ -1,7 +1,8 @@ import logging +import pprint import shutil import tempfile -from typing import Optional +from typing import List, Optional import click @@ -14,6 +15,8 @@ from datahub.ingestion.source.source_registry import source_registry from datahub.ingestion.transformer.transform_registry import transform_registry from datahub.telemetry import telemetry +from datahub.testing.compare_metadata_json import diff_metadata_json, load_json_file +from datahub.testing.mcp_diff import MCPDiff logger = logging.getLogger(__name__) @@ -69,6 +72,48 @@ def metadata_file(json_file: str, rewrite: bool, unpack_mces: bool) -> None: shutil.copy(out_file.name, json_file) +@check.command(no_args_is_help=True) +@click.argument( + "actual-file", + type=click.Path(exists=True, dir_okay=False, readable=True), +) +@click.argument( + "expected-file", + type=click.Path(exists=True, dir_okay=False, readable=True), +) +@click.option( + "--verbose", + "-v", + type=bool, + default=False, + help="Print full aspects that were changed, when comparing MCPs", +) +@click.option( + "--ignore-path", + multiple=True, + type=str, + default=(), + help="[Advanced] Paths in the deepdiff object to ignore", +) +@telemetry.with_telemetry() +def metadata_diff( + actual_file: str, expected_file: str, verbose: bool, ignore_path: List[str] +) -> None: + """Compare two metadata (MCE or MCP) JSON files. + + Comparison is more sophisticated for files composed solely of MCPs. + """ + + actual = load_json_file(actual_file) + expected = load_json_file(expected_file) + + diff = diff_metadata_json(output=actual, golden=expected, ignore_paths=ignore_path) + if isinstance(diff, MCPDiff): + click.echo(diff.pretty(verbose=verbose)) + else: + click.echo(pprint.pformat(diff)) + + @check.command() @click.option( "--verbose", diff --git a/metadata-ingestion/src/datahub/emitter/aspect.py b/metadata-ingestion/src/datahub/emitter/aspect.py index d3dff1f900ff2..9118967a07273 100644 --- a/metadata-ingestion/src/datahub/emitter/aspect.py +++ b/metadata-ingestion/src/datahub/emitter/aspect.py @@ -9,3 +9,6 @@ for name, klass in ASPECT_MAP.items() if klass.get_aspect_type() == "timeseries" } + +JSON_CONTENT_TYPE = "application/json" +JSON_PATCH_CONTENT_TYPE = "application/json-patch+json" diff --git a/metadata-ingestion/src/datahub/emitter/mcp.py b/metadata-ingestion/src/datahub/emitter/mcp.py index 20e0c659ae283..6f9a22bffd085 100644 --- a/metadata-ingestion/src/datahub/emitter/mcp.py +++ b/metadata-ingestion/src/datahub/emitter/mcp.py @@ -2,7 +2,7 @@ import json from typing import TYPE_CHECKING, List, Optional, Tuple, Union -from datahub.emitter.aspect import ASPECT_MAP, TIMESERIES_ASPECT_MAP +from datahub.emitter.aspect import ASPECT_MAP, JSON_CONTENT_TYPE, TIMESERIES_ASPECT_MAP from datahub.emitter.serialization_helper import post_json_transform, pre_json_transform from datahub.metadata.schema_classes import ( ChangeTypeClass, @@ -20,14 +20,12 @@ _ENTITY_TYPE_UNSET = "ENTITY_TYPE_UNSET" -_ASPECT_CONTENT_TYPE = "application/json" - def _make_generic_aspect(codegen_obj: DictWrapper) -> GenericAspectClass: serialized = json.dumps(pre_json_transform(codegen_obj.to_obj())) return GenericAspectClass( value=serialized.encode(), - contentType=_ASPECT_CONTENT_TYPE, + contentType=JSON_CONTENT_TYPE, ) @@ -42,7 +40,7 @@ def _try_from_generic_aspect( return True, None assert aspectName is not None, "aspectName must be set if aspect is set" - if aspect.contentType != _ASPECT_CONTENT_TYPE: + if aspect.contentType != JSON_CONTENT_TYPE: return False, None if aspectName not in ASPECT_MAP: @@ -155,7 +153,7 @@ def to_obj(self, tuples: bool = False, simplified_structure: bool = False) -> di # Undo the double JSON serialization that happens in the MCP aspect. if ( obj.get("aspect") - and obj["aspect"].get("contentType") == _ASPECT_CONTENT_TYPE + and obj["aspect"].get("contentType") == JSON_CONTENT_TYPE ): obj["aspect"] = {"json": json.loads(obj["aspect"]["value"])} return obj @@ -174,7 +172,7 @@ def from_obj( # routine works. if obj.get("aspect") and obj["aspect"].get("json"): obj["aspect"] = { - "contentType": _ASPECT_CONTENT_TYPE, + "contentType": JSON_CONTENT_TYPE, "value": json.dumps(obj["aspect"]["json"]), } diff --git a/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py index a7c0e1c9c9935..be68d46472a55 100644 --- a/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from typing import Any, Dict, Iterable, List, Optional +from datahub.emitter.aspect import JSON_PATCH_CONTENT_TYPE from datahub.emitter.serialization_helper import pre_json_transform from datahub.metadata.schema_classes import ( ChangeTypeClass, @@ -72,7 +73,7 @@ def build(self) -> Iterable[MetadataChangeProposalClass]: value=json.dumps( pre_json_transform(_recursive_to_obj(patches)) ).encode(), - contentType="application/json-patch+json", + contentType=JSON_PATCH_CONTENT_TYPE, ), auditHeader=self.audit_header, systemMetadata=self.system_metadata, diff --git a/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py b/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py index c78e72ac52c59..da70364d9aaab 100644 --- a/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py +++ b/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py @@ -10,6 +10,7 @@ IgnorableError, redact_raw_config, ) +from datahub.emitter.aspect import JSON_CONTENT_TYPE from datahub.emitter.mce_builder import datahub_guid from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.mcp_builder import make_data_platform_urn @@ -205,7 +206,7 @@ def on_completion( structured_report = StructuredExecutionReportClass( type="CLI_INGEST", serializedValue=structured_report_str, - contentType="application/json", + contentType=JSON_CONTENT_TYPE, ) execution_result_aspect = ExecutionRequestResultClass( status=status, diff --git a/metadata-ingestion/src/datahub/ingestion/sink/file.py b/metadata-ingestion/src/datahub/ingestion/sink/file.py index 4d7b881cf780c..c4f34d780f7c8 100644 --- a/metadata-ingestion/src/datahub/ingestion/sink/file.py +++ b/metadata-ingestion/src/datahub/ingestion/sink/file.py @@ -4,6 +4,7 @@ from typing import Iterable, Union from datahub.configuration.common import ConfigModel +from datahub.emitter.aspect import JSON_CONTENT_TYPE, JSON_PATCH_CONTENT_TYPE from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.common import RecordEnvelope from datahub.ingestion.api.sink import Sink, SinkReport, WriteCallback @@ -27,6 +28,14 @@ def _to_obj_for_file( ) -> dict: if isinstance(obj, MetadataChangeProposalWrapper): return obj.to_obj(simplified_structure=simplified_structure) + elif isinstance(obj, MetadataChangeProposal) and simplified_structure: + serialized = obj.to_obj() + if serialized.get("aspect") and serialized["aspect"].get("contentType") in [ + JSON_CONTENT_TYPE, + JSON_PATCH_CONTENT_TYPE, + ]: + serialized["aspect"] = {"json": json.loads(serialized["aspect"]["value"])} + return serialized return obj.to_obj() @@ -82,6 +91,7 @@ def write_metadata_file( MetadataChangeProposal, MetadataChangeProposalWrapper, UsageAggregation, + dict, # Serialized MCE or MCP ] ], ) -> None: @@ -91,6 +101,7 @@ def write_metadata_file( for i, record in enumerate(records): if i > 0: f.write(",\n") - obj = _to_obj_for_file(record) - json.dump(obj, f, indent=4) + if not isinstance(record, dict): + record = _to_obj_for_file(record) + json.dump(record, f, indent=4) f.write("\n]") diff --git a/metadata-ingestion/src/datahub/testing/compare_metadata_json.py b/metadata-ingestion/src/datahub/testing/compare_metadata_json.py new file mode 100644 index 0000000000000..30a6db6871873 --- /dev/null +++ b/metadata-ingestion/src/datahub/testing/compare_metadata_json.py @@ -0,0 +1,109 @@ +"""Utilities for comparing MCE and MCP files.""" + +import json +import logging +import os +import pathlib +import pprint +import shutil +import tempfile +from typing import Any, Dict, List, Sequence, Union + +import pytest +from deepdiff import DeepDiff + +from datahub.ingestion.sink.file import write_metadata_file +from datahub.ingestion.source.file import read_metadata_file +from datahub.testing.mcp_diff import MCPDiff, get_aspects_by_urn + +logger = logging.getLogger(__name__) + +MetadataJson = List[Dict[str, Any]] + +default_exclude_paths = [ + r"root\[\d+\]\['systemMetadata'\]\['lastObserved'\]", + r"root\[\d+\]\['aspect'\]\['json'\]\['timestampMillis'\]", + r"root\[\d+\]\['aspect'\]\['json'\]\['lastUpdatedTimestamp'\]", + r"root\[\d+\]\['aspect'\]\['json'\]\['created'\]", + r"root\[\d+\]\['aspect'\]\['json'\]\['lastModified'\]", +] + + +def load_json_file(filename: Union[str, os.PathLike]) -> MetadataJson: + with open(str(filename)) as f: + return json.load(f) + + +def assert_metadata_files_equal( + output_path: Union[str, os.PathLike], + golden_path: Union[str, os.PathLike], + update_golden: bool, + copy_output: bool, + ignore_paths: Sequence[str] = (), +) -> None: + golden_exists = os.path.isfile(golden_path) + + if copy_output: + shutil.copyfile(str(output_path), str(golden_path) + ".output") + print(f"Copied output file to {golden_path}.output") + + if not update_golden and not golden_exists: + raise FileNotFoundError( + "Golden file does not exist. Please run with the --update-golden-files option to create." + ) + + output = load_json_file(output_path) + + if update_golden and not golden_exists: + golden = load_json_file(output_path) + shutil.copyfile(str(output_path), str(golden_path)) + return + else: + # We have to "normalize" the golden file by reading and writing it back out. + # This will clean up nulls, double serialization, and other formatting issues. + with tempfile.NamedTemporaryFile() as temp: + golden_metadata = read_metadata_file(pathlib.Path(golden_path)) + write_metadata_file(pathlib.Path(temp.name), golden_metadata) + golden = load_json_file(temp.name) + + diff = diff_metadata_json(output, golden, ignore_paths) + if diff and update_golden: + if isinstance(diff, MCPDiff): + diff.apply_delta(golden) + write_metadata_file(pathlib.Path(golden_path), golden) + else: + shutil.copyfile(str(output_path), str(golden_path)) + return + + if diff: + # Call pytest.fail rather than raise an exception to omit stack trace + if isinstance(diff, MCPDiff): + print(diff.pretty(verbose=True)) + pytest.fail(diff.pretty(), pytrace=False) + else: + pytest.fail(pprint.pformat(diff), pytrace=False) + + +def diff_metadata_json( + output: MetadataJson, + golden: MetadataJson, + ignore_paths: Sequence[str] = (), +) -> Union[DeepDiff, MCPDiff]: + ignore_paths = (*ignore_paths, *default_exclude_paths) + try: + golden_map = get_aspects_by_urn(golden) + output_map = get_aspects_by_urn(output) + return MCPDiff.create( + golden=golden_map, + output=output_map, + ignore_paths=ignore_paths, + ) + except AssertionError as e: + logger.warning(f"Reverting to old diff method: {e}") + logger.debug("Error with new diff method", exc_info=True) + return DeepDiff( + golden, + output, + exclude_regex_paths=ignore_paths, + ignore_order=True, + ) diff --git a/metadata-ingestion/src/datahub/testing/mcp_diff.py b/metadata-ingestion/src/datahub/testing/mcp_diff.py new file mode 100644 index 0000000000000..f7aeb6f829012 --- /dev/null +++ b/metadata-ingestion/src/datahub/testing/mcp_diff.py @@ -0,0 +1,266 @@ +import re +from collections import defaultdict +from dataclasses import dataclass, field +from typing import Any, Dict, List, Sequence, Set, Tuple, Union + +import deepdiff.serialization +import yaml +from deepdiff import DeepDiff +from deepdiff.model import DiffLevel +from deepdiff.operator import BaseOperator +from typing_extensions import Literal + +ReportType = Literal[ + "type_changes", + "dictionary_item_added", + "dictionary_item_removed", + "values_changed", + "unprocessed", + "iterable_item_added", + "iterable_item_removed", + "iterable_item_moved", + "attribute_added", + "attribute_removed", + "set_item_added", + "set_item_removed", + "repetition_change", +] + + +@dataclass(frozen=True) +class AspectForDiff: + urn: str + change_type: str + aspect_name: str + aspect: Dict[str, Any] = field(hash=False) + delta_info: "DeltaInfo" = field(hash=False) + + @classmethod + def create_from_mcp(cls, idx: int, obj: Dict[str, Any]) -> "AspectForDiff": + aspect = obj["aspect"] + return cls( + urn=obj["entityUrn"], + change_type=obj["changeType"], + aspect_name=obj["aspectName"], + aspect=aspect.get("json", aspect), + delta_info=DeltaInfo(idx=idx, original=obj), + ) + + +@dataclass +class DeltaInfo: + """Information about an MCP used to construct a diff delta. + + In a separate class so it can be ignored by DeepDiff via MCPDeltaInfoOperator. + """ + + idx: int # Location in list of MCEs in golden file + original: Dict[str, Any] # Original json-serialized MCP + + +class DeltaInfoOperator(BaseOperator): + def __init__(self): + super().__init__(types=[DeltaInfo]) + + def give_up_diffing(self, *args: Any, **kwargs: Any) -> bool: + return True + + +AspectsByUrn = Dict[str, Dict[str, List[AspectForDiff]]] + + +def get_aspects_by_urn(obj: object) -> AspectsByUrn: + """Restructure a list of serialized MCPs by urn and aspect. + Retains information like the original dict and index to facilitate `apply_delta` later. + + Raises: + AssertionError: If the input is not purely a list of MCPs. + """ + d: AspectsByUrn = defaultdict(dict) + assert isinstance(obj, list), obj + for i, entry in enumerate(obj): + assert isinstance(entry, dict), entry + if "proposedSnapshot" in entry: + raise AssertionError("Found MCEs in output") + elif "entityUrn" in entry and "aspectName" in entry and "aspect" in entry: + urn = entry["entityUrn"] + aspect_name = entry["aspectName"] + aspect = AspectForDiff.create_from_mcp(i, entry) + d[urn].setdefault(aspect_name, []).append(aspect) + else: + raise AssertionError(f"Unrecognized MCE: {entry}") + + return d + + +@dataclass +class MCPAspectDiff: + aspects_added: Dict[int, AspectForDiff] + aspects_removed: Dict[int, AspectForDiff] + aspects_changed: Dict[Tuple[int, AspectForDiff, AspectForDiff], List[DiffLevel]] + + @classmethod + def create(cls, diff: DeepDiff) -> "MCPAspectDiff": + # Parse DeepDiff to distinguish between aspects that were added, removed, or changed + aspects_added = {} + aspects_removed = {} + aspects_changed = defaultdict(list) + for key, diff_levels in diff.tree.items(): + for diff_level in diff_levels: + path = diff_level.path(output_format="list") + idx = int(path[0]) + if len(path) == 1 and key == "iterable_item_added": + aspects_added[idx] = diff_level.t2 + elif len(path) == 1 and key == "iterable_item_removed": + aspects_removed[idx] = diff_level.t1 + else: + level = diff_level + while not isinstance(level.t1, AspectForDiff): + level = level.up + aspects_changed[(idx, level.t1, level.t2)].append(diff_level) + + return cls( + aspects_added=aspects_added, + aspects_removed=aspects_removed, + aspects_changed=aspects_changed, + ) + + +@dataclass +class MCPDiff: + aspect_changes: Dict[str, Dict[str, MCPAspectDiff]] # urn -> aspect -> diff + urns_added: Set[str] + urns_removed: Set[str] + + def __bool__(self) -> bool: + return bool(self.aspect_changes) + + @classmethod + def create( + cls, + golden: AspectsByUrn, + output: AspectsByUrn, + ignore_paths: Sequence[str], + ) -> "MCPDiff": + ignore_paths = [cls.convert_path(path) for path in ignore_paths] + + aspect_changes: Dict[str, Dict[str, MCPAspectDiff]] = defaultdict(dict) + for urn in golden.keys() | output.keys(): + golden_map = golden.get(urn, {}) + output_map = output.get(urn, {}) + for aspect_name in golden_map.keys() | output_map.keys(): + diff = DeepDiff( + t1=golden_map.get(aspect_name, []), + t2=output_map.get(aspect_name, []), + exclude_regex_paths=ignore_paths, + ignore_order=True, + custom_operators=[DeltaInfoOperator()], + ) + if diff: + aspect_changes[urn][aspect_name] = MCPAspectDiff.create(diff) + + return cls( + urns_added=output.keys() - golden.keys(), + urns_removed=golden.keys() - output.keys(), + aspect_changes=aspect_changes, + ) + + @staticmethod + def convert_path(path: str) -> str: + # Attempt to use paths intended for the root golden... sorry for the regex + return re.sub( + r"root\\?\[([0-9]+|\\d\+)\\?]\\?\['aspect'\\?](\\?\['(json|value)'\\?])?", + r"root\[\\d+].aspect", + path, + ) + + def apply_delta(self, golden: List[Dict[str, Any]]) -> None: + aspect_diffs = [v for d in self.aspect_changes.values() for v in d.values()] + for aspect_diff in aspect_diffs: + for (_, old, new), diffs in aspect_diff.aspects_changed.items(): + golden[old.delta_info.idx] = new.delta_info.original + + indices_to_remove = set() + for aspect_diff in aspect_diffs: + for ga in aspect_diff.aspects_removed.values(): + indices_to_remove.add(ga.delta_info.idx) + for idx in sorted(indices_to_remove, reverse=True): + del golden[idx] + + for aspect_diff in aspect_diffs: # Ideally would have smarter way to do this + for ga in aspect_diff.aspects_added.values(): + golden.insert(ga.delta_info.idx, ga.delta_info.original) + + def pretty(self, verbose: bool = False) -> str: + """The pretty human-readable string output of the diff between golden and output.""" + s = [] + for urn in self.urns_added: + s.append(f"Urn added, {urn}{' with aspects:' if verbose else ''}") + if verbose: + for aspect_diff in self.aspect_changes[urn].values(): + for i, ga in aspect_diff.aspects_added.items(): + s.append(self.report_aspect(ga, i)) + s.append(serialize_aspect(ga.aspect)) + if self.urns_added: + s.append("") + + for urn in self.urns_removed: + s.append(f"Urn removed, {urn}{' with aspects:' if verbose else ''}") + if verbose: + for aspect_diff in self.aspect_changes[urn].values(): + for i, ga in aspect_diff.aspects_removed.items(): + s.append(self.report_aspect(ga, i)) + s.append(serialize_aspect(ga.aspect)) + if self.urns_removed: + s.append("") + + for urn in self.aspect_changes.keys() - self.urns_added - self.urns_removed: + aspect_map = self.aspect_changes[urn] + s.append(f"Urn changed, {urn}:") + for aspect_name, aspect_diffs in aspect_map.items(): + for i, ga in aspect_diffs.aspects_added.items(): + s.append(self.report_aspect(ga, i, "added")) + if verbose: + s.append(serialize_aspect(ga.aspect)) + for i, ga in aspect_diffs.aspects_removed.items(): + s.append(self.report_aspect(ga, i, "removed")) + if verbose: + s.append(serialize_aspect(ga.aspect)) + for (i, old, new), diffs in aspect_diffs.aspects_changed.items(): + s.append(self.report_aspect(old, i, "changed") + ":") + for diff_level in diffs: + s.append(self.report_diff_level(diff_level, i)) + if verbose: + s.append(f"Old aspect:\n{serialize_aspect(old.aspect)}") + s.append(f"New aspect:\n{serialize_aspect(new.aspect)}") + + s.append("") + + return "\n".join(s) + + @staticmethod + def report_aspect(ga: AspectForDiff, idx: int, msg: str = "") -> str: + # Describe as "nth " if n > 1 + base = (idx + 1) % 10 + if base == 1: + suffix = "st" + elif base == 2: + suffix = "nd" + elif base == 3: + suffix = "rd" + else: + suffix = "th" + ordinal = f"{(idx+1)}{suffix} " if idx else "" + return f"{ordinal}<{ga.aspect_name}> {msg}" + + @staticmethod + def report_diff_level(diff: DiffLevel, idx: int) -> str: + return "\t" + deepdiff.serialization.pretty_print_diff(diff).replace( + f"root[{idx}].", "" + ) + + +def serialize_aspect(aspect: Union[AspectForDiff, Dict[str, Any]]) -> str: + if isinstance(aspect, AspectForDiff): # Unpack aspect + aspect = aspect.aspect + return " " + yaml.dump(aspect, sort_keys=False).replace("\n", "\n ").strip() diff --git a/metadata-ingestion/tests/test_helpers/mce_helpers.py b/metadata-ingestion/tests/test_helpers/mce_helpers.py index 3914a7f77a55d..46bf5cbea6edf 100644 --- a/metadata-ingestion/tests/test_helpers/mce_helpers.py +++ b/metadata-ingestion/tests/test_helpers/mce_helpers.py @@ -1,19 +1,26 @@ import json import logging import os -import pathlib -import pprint import re -import shutil -import tempfile -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union - -import deepdiff +from typing import ( + Any, + Callable, + Dict, + List, + Optional, + Sequence, + Set, + Tuple, + Type, + Union, +) from datahub.emitter.mcp import MetadataChangeProposalWrapper -from datahub.ingestion.sink.file import write_metadata_file -from datahub.ingestion.source.file import read_metadata_file from datahub.metadata.schema_classes import MetadataChangeEventClass +from datahub.testing.compare_metadata_json import ( + assert_metadata_files_equal, + load_json_file, +) from datahub.utilities.urns.urn import Urn from tests.test_helpers.type_helpers import PytestConfig @@ -21,9 +28,6 @@ IGNORE_PATH_TIMESTAMPS = [ # Ignore timestamps from the ETL pipeline. A couple examples: - # root[0]['proposedSnapshot']['com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot']['aspects'][0]['com.linkedin.pegasus2avro.common.Ownership']['lastModified']['time'] - # root[69]['proposedSnapshot']['com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot']['aspects'][0]['com.linkedin.pegasus2avro.schema.SchemaMetadata']['lastModified']['time']" - # root[0]['proposedSnapshot']['com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot']['aspects'][1]['com.linkedin.pegasus2avro.dataset.UpstreamLineage']['upstreams'][0]['auditStamp']['time'] r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['created'\]\['time'\]", r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['lastModified'\]\['time'\]", r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['createStamp'\]\['time'\]", @@ -56,12 +60,6 @@ class EntityType: GROUP = "corpGroup" -def load_json_file(filename: Union[str, os.PathLike]) -> object: - with open(str(filename)) as f: - a = json.load(f) - return a - - def clean_nones(value): """ Recursively remove all None values from dictionaries and lists, and returns @@ -75,92 +73,21 @@ def clean_nones(value): return value -def assert_mces_equal( - output: object, golden: object, ignore_paths: Optional[List[str]] = None -) -> None: - # This method assumes we're given a list of MCE json objects. - diff = deepdiff.DeepDiff( - golden, output, exclude_regex_paths=ignore_paths, ignore_order=True - ) - if diff: - # Attempt a clean diff (removing None-s) - assert isinstance(output, list) - assert isinstance(golden, list) - clean_output = [clean_nones(o) for o in output] - clean_golden = [clean_nones(g) for g in golden] - clean_diff = deepdiff.DeepDiff( - clean_golden, - clean_output, - exclude_regex_paths=ignore_paths, - ignore_order=True, - ) - if not clean_diff: - logger.debug(f"MCE-s differ, clean MCE-s are fine\n{pprint.pformat(diff)}") - diff = clean_diff - if diff: - # do some additional processing to emit helpful messages - output_urns = _get_entity_urns(output) - golden_urns = _get_entity_urns(golden) - in_golden_but_not_in_output = golden_urns - output_urns - in_output_but_not_in_golden = output_urns - golden_urns - if in_golden_but_not_in_output: - logger.info( - f"Golden file has {len(in_golden_but_not_in_output)} more urns: {in_golden_but_not_in_output}" - ) - if in_output_but_not_in_golden: - logger.info( - f"Golden file has {len(in_output_but_not_in_golden)} more urns: {in_output_but_not_in_golden}" - ) - - assert ( - not diff - ), f"MCEs differ\n{pprint.pformat(diff)} \n output was: {json.dumps(output)}" - - def check_golden_file( pytestconfig: PytestConfig, output_path: Union[str, os.PathLike], golden_path: Union[str, os.PathLike], - ignore_paths: Optional[List[str]] = None, + ignore_paths: Sequence[str] = (), ) -> None: update_golden = pytestconfig.getoption("--update-golden-files") copy_output = pytestconfig.getoption("--copy-output-files") - golden_exists = os.path.isfile(golden_path) - - if copy_output: - shutil.copyfile(str(output_path), str(golden_path) + ".output") - print(f"Copied output file to {golden_path}.output") - - if not update_golden and not golden_exists: - raise FileNotFoundError( - "Golden file does not exist. Please run with the --update-golden-files option to create." - ) - - output = load_json_file(output_path) - - # if updating a golden file that doesn't exist yet, load the output again - if update_golden and not golden_exists: - golden = load_json_file(output_path) - shutil.copyfile(str(output_path), str(golden_path)) - else: - # We have to "normalize" the golden file by reading and writing it back out. - # This will clean up nulls, double serialization, and other formatting issues. - with tempfile.NamedTemporaryFile() as temp: - golden_metadata = read_metadata_file(pathlib.Path(golden_path)) - write_metadata_file(pathlib.Path(temp.name), golden_metadata) - golden = load_json_file(temp.name) - - try: - assert_mces_equal(output, golden, ignore_paths) - - except AssertionError as e: - # only update golden files if the diffs are not empty - if update_golden: - shutil.copyfile(str(output_path), str(golden_path)) - - # raise the error if we're just running the test - else: - raise e + assert_metadata_files_equal( + output_path=output_path, + golden_path=golden_path, + update_golden=update_golden, + copy_output=copy_output, + ignore_paths=ignore_paths, + ) def _get_field_for_entity_type_in_mce(entity_type: str) -> str: diff --git a/metadata-ingestion/tests/unit/patch/complex_dataset_patch.json b/metadata-ingestion/tests/unit/patch/complex_dataset_patch.json index 08e100140c38b..d5dfe125942fb 100644 --- a/metadata-ingestion/tests/unit/patch/complex_dataset_patch.json +++ b/metadata-ingestion/tests/unit/patch/complex_dataset_patch.json @@ -1,42 +1,83 @@ [ - { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", - "changeType": "PATCH", - "aspectName": "datasetProperties", - "aspect": { - "value": "[{\"op\": \"replace\", \"path\": \"/description\", \"value\": \"test description\"}, {\"op\": \"add\", \"path\": \"/customProperties/test_key_1\", \"value\": \"test_value_1\"}, {\"op\": \"add\", \"path\": \"/customProperties/test_key_2\", \"value\": \"test_value_2\"}]", - "contentType": "application/json-patch+json" - } - }, - { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", - "changeType": "PATCH", - "aspectName": "globalTags", - "aspect": { - "value": "[{\"op\": \"add\", \"path\": \"/tags/urn:li:tag:test_tag\", \"value\": {\"tag\": \"urn:li:tag:test_tag\"}}]", - "contentType": "application/json-patch+json" - } - }, - { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", - "changeType": "PATCH", - "aspectName": "upstreamLineage", - "aspect": { - "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cfct_users_created_upstream%2CPROD%29\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created_upstream,PROD)\", \"type\": \"TRANSFORMED\"}}]", - "contentType": "application/json-patch+json" - } - }, - { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", - "changeType": "PATCH", - "aspectName": "editableSchemaMetadata", - "aspect": { - "value": "[{\"op\": \"add\", \"path\": \"/editableSchemaFieldInfo/field1/globalTags/tags/urn:li:tag:tag1\", \"value\": {\"tag\": \"urn:li:tag:tag1\"}}]", - "contentType": "application/json-patch+json" - } - } +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "changeType": "PATCH", + "aspectName": "datasetProperties", + "aspect": { + "json": [ + { + "op": "replace", + "path": "/description", + "value": "test description" + }, + { + "op": "add", + "path": "/customProperties/test_key_1", + "value": "test_value_1" + }, + { + "op": "add", + "path": "/customProperties/test_key_2", + "value": "test_value_2" + } + ] + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "changeType": "PATCH", + "aspectName": "globalTags", + "aspect": { + "json": [ + { + "op": "add", + "path": "/tags/urn:li:tag:test_tag", + "value": { + "tag": "urn:li:tag:test_tag" + } + } + ] + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "json": [ + { + "op": "add", + "path": "/upstreams/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2Cfct_users_created_upstream%2CPROD%29", + "value": { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created_upstream,PROD)", + "type": "TRANSFORMED" + } + } + ] + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "changeType": "PATCH", + "aspectName": "editableSchemaMetadata", + "aspect": { + "json": [ + { + "op": "add", + "path": "/editableSchemaFieldInfo/field1/globalTags/tags/urn:li:tag:tag1", + "value": { + "tag": "urn:li:tag:tag1" + } + } + ] + } +} ] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/test_bigquery_usage.py b/metadata-ingestion/tests/unit/test_bigquery_usage.py index 01a7b34f3de64..e06c6fb3fe7e5 100644 --- a/metadata-ingestion/tests/unit/test_bigquery_usage.py +++ b/metadata-ingestion/tests/unit/test_bigquery_usage.py @@ -34,10 +34,10 @@ OperationClass, TimeWindowSizeClass, ) +from datahub.testing.compare_metadata_json import diff_metadata_json from tests.performance.bigquery import generate_events, ref_from_table from tests.performance.data_generation import generate_data, generate_queries from tests.performance.data_model import Container, FieldAccess, Query, Table, View -from tests.test_helpers.mce_helpers import assert_mces_equal PROJECT_1 = "project-1" PROJECT_2 = "project-2" @@ -224,7 +224,7 @@ def make_zero_usage_workunit( def compare_workunits( output: Iterable[MetadataWorkUnit], expected: Iterable[MetadataWorkUnit] ) -> None: - assert_mces_equal( + assert not diff_metadata_json( [wu.metadata.to_obj() for wu in output], [wu.metadata.to_obj() for wu in expected], ) diff --git a/metadata-ingestion/tests/unit/test_mce_helpers.py b/metadata-ingestion/tests/unit/test_compare_metadata.py similarity index 96% rename from metadata-ingestion/tests/unit/test_mce_helpers.py rename to metadata-ingestion/tests/unit/test_compare_metadata.py index 5af991755fdb9..8316e226d4847 100644 --- a/metadata-ingestion/tests/unit/test_mce_helpers.py +++ b/metadata-ingestion/tests/unit/test_compare_metadata.py @@ -2,6 +2,7 @@ import pytest +from datahub.testing.compare_metadata_json import diff_metadata_json from tests.test_helpers import mce_helpers basic_1 = json.loads( @@ -149,18 +150,18 @@ def test_basic_diff_same() -> None: - mce_helpers.assert_mces_equal(basic_1, basic_2, mce_helpers.IGNORE_PATH_TIMESTAMPS) + assert not diff_metadata_json(basic_1, basic_2, mce_helpers.IGNORE_PATH_TIMESTAMPS) def test_basic_diff_only_owner_change() -> None: with pytest.raises(AssertionError): - mce_helpers.assert_mces_equal( + assert not diff_metadata_json( basic_2, basic_3, mce_helpers.IGNORE_PATH_TIMESTAMPS ) def test_basic_diff_owner_change() -> None: with pytest.raises(AssertionError): - mce_helpers.assert_mces_equal( + assert not diff_metadata_json( basic_1, basic_3, mce_helpers.IGNORE_PATH_TIMESTAMPS ) diff --git a/metadata-ingestion/tests/unit/test_generic_aspect_transformer.py b/metadata-ingestion/tests/unit/test_generic_aspect_transformer.py index fe05ac49a9228..18b0d9fd40041 100644 --- a/metadata-ingestion/tests/unit/test_generic_aspect_transformer.py +++ b/metadata-ingestion/tests/unit/test_generic_aspect_transformer.py @@ -2,6 +2,7 @@ import unittest from typing import Any, List, Optional +from datahub.emitter.aspect import JSON_CONTENT_TYPE from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.common import EndOfStream, PipelineContext, RecordEnvelope from datahub.ingestion.transformer.generic_aspect_transformer import ( @@ -94,7 +95,7 @@ def transform_generic_aspect( aspect.value if aspect else json.dumps({"customAspect": 10}).encode("utf-8") ) result_aspect = GenericAspectClass( - contentType="application/json", + contentType=JSON_CONTENT_TYPE, value=value, ) return result_aspect @@ -183,7 +184,7 @@ def test_modify_generic_aspect_when_mcpc_received(self): entity_urn="urn:li:dataset:(urn:li:dataPlatform:snowflake,example1,PROD)", aspect_name="customAspect", aspect=GenericAspectClass( - contentType="application/json", + contentType=JSON_CONTENT_TYPE, value=json.dumps({"customAspect": 5}).encode("utf-8"), ), ) @@ -251,7 +252,7 @@ def test_remove_generic_aspect_when_mcpc_received(self): entity_urn="urn:li:dataset:(urn:li:dataPlatform:snowflake,example1,PROD)", aspect_name="customAspect", aspect=GenericAspectClass( - contentType="application/json", + contentType=JSON_CONTENT_TYPE, value=json.dumps({"customAspect": 5}).encode("utf-8"), ), ) diff --git a/metadata-ingestion/tests/unit/test_workunit.py b/metadata-ingestion/tests/unit/test_workunit.py index 9d31b3fc60866..5a4fbf315edb6 100644 --- a/metadata-ingestion/tests/unit/test_workunit.py +++ b/metadata-ingestion/tests/unit/test_workunit.py @@ -1,5 +1,6 @@ import json +from datahub.emitter.aspect import JSON_CONTENT_TYPE, JSON_PATCH_CONTENT_TYPE from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.metadata.schema_classes import ( @@ -48,7 +49,7 @@ def test_get_aspects_of_type_mcpc(): aspectName=StatusClass.ASPECT_NAME, aspect=GenericAspectClass( value=json.dumps(aspect.to_obj()).encode(), - contentType="application/json", + contentType=JSON_CONTENT_TYPE, ), ) wu = MetadataWorkUnit(id="id", mcp_raw=mcpc) @@ -63,7 +64,7 @@ def test_get_aspects_of_type_mcpc(): aspectName="not status", aspect=GenericAspectClass( value=json.dumps(aspect.to_obj()).encode(), - contentType="application/json", + contentType=JSON_CONTENT_TYPE, ), ) wu = MetadataWorkUnit(id="id", mcp_raw=mcpc) @@ -76,7 +77,7 @@ def test_get_aspects_of_type_mcpc(): aspectName=StatusClass.ASPECT_NAME, aspect=GenericAspectClass( value=json.dumps({"not_status": True}).encode(), - contentType="application/json-patch+json", + contentType=JSON_PATCH_CONTENT_TYPE, ), ) wu = MetadataWorkUnit(id="id", mcp_raw=mcpc) @@ -89,7 +90,7 @@ def test_get_aspects_of_type_mcpc(): aspectName=StatusClass.ASPECT_NAME, aspect=GenericAspectClass( value=(json.dumps(aspect.to_obj()) + "aaa").encode(), - contentType="application/json", + contentType=JSON_CONTENT_TYPE, ), ) wu = MetadataWorkUnit(id="id", mcp_raw=mcpc) @@ -102,7 +103,7 @@ def test_get_aspects_of_type_mcpc(): aspectName=StatusClass.ASPECT_NAME, aspect=GenericAspectClass( value='{"ß": 2}'.encode("latin_1"), - contentType="application/json", + contentType=JSON_CONTENT_TYPE, ), ) wu = MetadataWorkUnit(id="id", mcp_raw=mcpc) From 759d0c86af240742fb78dd0df34502118f81791e Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Tue, 11 Jul 2023 12:55:32 -0400 Subject: [PATCH 121/222] test(ingest/airflow): Fix test for airflow 2.6.3 (#8393) --- .../src/datahub_provider/_airflow_shims.py | 3 +++ metadata-ingestion/tests/unit/test_airflow.py | 25 +++++++++---------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/metadata-ingestion/src/datahub_provider/_airflow_shims.py b/metadata-ingestion/src/datahub_provider/_airflow_shims.py index 24917c982dc95..31e1237c0d21d 100644 --- a/metadata-ingestion/src/datahub_provider/_airflow_shims.py +++ b/metadata-ingestion/src/datahub_provider/_airflow_shims.py @@ -5,12 +5,14 @@ try: from airflow.models.mappedoperator import MappedOperator from airflow.models.operator import Operator + from airflow.operators.empty import EmptyOperator except ModuleNotFoundError: # Operator isn't a real class, but rather a type alias defined # as the union of BaseOperator and MappedOperator. # Since older versions of Airflow don't have MappedOperator, we can just use BaseOperator. Operator = BaseOperator # type: ignore MappedOperator = None # type: ignore + from airflow.operators.dummy import DummyOperator as EmptyOperator # type: ignore try: from airflow.sensors.external_task import ExternalTaskSensor @@ -22,5 +24,6 @@ __all__ = [ "Operator", "MappedOperator", + "EmptyOperator", "ExternalTaskSensor", ] diff --git a/metadata-ingestion/tests/unit/test_airflow.py b/metadata-ingestion/tests/unit/test_airflow.py index 8cbe4752938bc..980dc5550fafa 100644 --- a/metadata-ingestion/tests/unit/test_airflow.py +++ b/metadata-ingestion/tests/unit/test_airflow.py @@ -1,5 +1,3 @@ -from datahub_provider._airflow_compat import AIRFLOW_PATCHED - import datetime import json import os @@ -15,10 +13,10 @@ import pytest from airflow.lineage import apply_lineage, prepare_lineage from airflow.models import DAG, Connection, DagBag, DagRun, TaskInstance -from airflow.operators.dummy import DummyOperator import datahub.emitter.mce_builder as builder from datahub_provider import get_provider_info +from datahub_provider._airflow_shims import AIRFLOW_PATCHED, EmptyOperator from datahub_provider.entities import Dataset, Urn from datahub_provider.hooks.datahub import DatahubKafkaHook, DatahubRestHook from datahub_provider.operators.datahub import DatahubEmitterOperator @@ -238,12 +236,12 @@ def test_lineage_backend(mock_emit, inlets, outlets, capture_executions): dag = DAG(dag_id="test_lineage_is_sent_to_backend", start_date=DEFAULT_DATE) with dag: - op1 = DummyOperator( + op1 = EmptyOperator( task_id="task1_upstream", inlets=inlets, outlets=outlets, ) - op2 = DummyOperator( + op2 = EmptyOperator( task_id="task2", inlets=inlets, outlets=outlets, @@ -257,13 +255,14 @@ def test_lineage_backend(mock_emit, inlets, outlets, capture_executions): if AIRFLOW_VERSION < packaging.version.parse("2.2.0"): ti = TaskInstance(task=op2, execution_date=DEFAULT_DATE) # Ignoring type here because DagRun state is just a sring at Airflow 1 - dag_run = DagRun(state="success", run_id=f"scheduled_{DEFAULT_DATE}") # type: ignore + dag_run = DagRun(state="success", run_id=f"scheduled_{DEFAULT_DATE.isoformat()}") # type: ignore else: from airflow.utils.state import DagRunState ti = TaskInstance(task=op2, run_id=f"test_airflow-{DEFAULT_DATE}") dag_run = DagRun( - state=DagRunState.SUCCESS, run_id=f"scheduled_{DEFAULT_DATE}" + state=DagRunState.SUCCESS, + run_id=f"scheduled_{DEFAULT_DATE.isoformat()}", ) ti.dag_run = dag_run # type: ignore @@ -372,7 +371,7 @@ def test_lineage_backend(mock_emit, inlets, outlets, capture_executions): ) assert ( mock_emitter.method_calls[9].args[0].entityUrn - == "urn:li:dataProcessInstance:b6375e5f5faeb543cfb5d7d8a47661fb" + == "urn:li:dataProcessInstance:5e274228107f44cc2dd7c9782168cc29" ) assert ( @@ -381,7 +380,7 @@ def test_lineage_backend(mock_emit, inlets, outlets, capture_executions): ) assert ( mock_emitter.method_calls[10].args[0].entityUrn - == "urn:li:dataProcessInstance:b6375e5f5faeb543cfb5d7d8a47661fb" + == "urn:li:dataProcessInstance:5e274228107f44cc2dd7c9782168cc29" ) assert ( mock_emitter.method_calls[11].args[0].aspectName @@ -389,7 +388,7 @@ def test_lineage_backend(mock_emit, inlets, outlets, capture_executions): ) assert ( mock_emitter.method_calls[11].args[0].entityUrn - == "urn:li:dataProcessInstance:b6375e5f5faeb543cfb5d7d8a47661fb" + == "urn:li:dataProcessInstance:5e274228107f44cc2dd7c9782168cc29" ) assert ( mock_emitter.method_calls[12].args[0].aspectName @@ -397,7 +396,7 @@ def test_lineage_backend(mock_emit, inlets, outlets, capture_executions): ) assert ( mock_emitter.method_calls[12].args[0].entityUrn - == "urn:li:dataProcessInstance:b6375e5f5faeb543cfb5d7d8a47661fb" + == "urn:li:dataProcessInstance:5e274228107f44cc2dd7c9782168cc29" ) assert mock_emitter.method_calls[13].args[0].aspectName == "status" assert ( @@ -415,7 +414,7 @@ def test_lineage_backend(mock_emit, inlets, outlets, capture_executions): ) assert ( mock_emitter.method_calls[15].args[0].entityUrn - == "urn:li:dataProcessInstance:b6375e5f5faeb543cfb5d7d8a47661fb" + == "urn:li:dataProcessInstance:5e274228107f44cc2dd7c9782168cc29" ) assert ( mock_emitter.method_calls[16].args[0].aspectName @@ -423,5 +422,5 @@ def test_lineage_backend(mock_emit, inlets, outlets, capture_executions): ) assert ( mock_emitter.method_calls[16].args[0].entityUrn - == "urn:li:dataProcessInstance:b6375e5f5faeb543cfb5d7d8a47661fb" + == "urn:li:dataProcessInstance:5e274228107f44cc2dd7c9782168cc29" ) From d4135d57b713767cff1f83e4ad6ed1b09b2b075b Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 11 Jul 2023 11:12:51 -0700 Subject: [PATCH 122/222] feat(ingest/bigquery): support column-level lineage (#8382) --- .../src/datahub/ingestion/graph/client.py | 5 +- .../ingestion/source/bigquery_v2/bigquery.py | 98 ++-- .../source/bigquery_v2/bigquery_config.py | 19 +- .../source/bigquery_v2/bigquery_report.py | 5 + .../source/bigquery_v2/bigquery_schema.py | 2 +- .../ingestion/source/bigquery_v2/lineage.py | 421 ++++++++++++------ .../ingestion/source_report/sql/bigquery.py | 39 -- .../source_report/usage/bigquery_usage.py | 36 -- .../testing/check_sql_parser_result.py | 3 - .../utilities/file_backed_collections.py | 37 +- .../src/datahub/utilities/sqlglot_lineage.py | 67 ++- .../test_bigquery_star_with_replace.json | 47 ++ .../test_bigquery_view_from_union.json | 44 ++ .../unit/sql_parsing/test_sqlglot_lineage.py | 45 ++ .../tests/unit/test_bigquery_lineage.py | 94 +--- .../tests/unit/test_bigquery_source.py | 96 +++- 16 files changed, 662 insertions(+), 396 deletions(-) delete mode 100644 metadata-ingestion/src/datahub/ingestion/source_report/sql/bigquery.py delete mode 100644 metadata-ingestion/src/datahub/ingestion/source_report/usage/bigquery_usage.py create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_star_with_replace.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_view_from_union.json diff --git a/metadata-ingestion/src/datahub/ingestion/graph/client.py b/metadata-ingestion/src/datahub/ingestion/graph/client.py index ba07ea70c9656..2f817ee69a637 100644 --- a/metadata-ingestion/src/datahub/ingestion/graph/client.py +++ b/metadata-ingestion/src/datahub/ingestion/graph/client.py @@ -991,12 +991,15 @@ def parse_sql_lineage( return sqlglot_lineage( sql, - platform=platform, schema_resolver=schema_resolver, default_db=default_db, default_schema=default_schema, ) + def close(self) -> None: + self._make_schema_resolver.cache_clear() + super().close() + def get_default_graph() -> DataHubGraph: (url, token) = get_url_and_token() diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py index 82e9647c487bb..1e4a293239942 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py @@ -1,11 +1,12 @@ import atexit +import hashlib import logging import os import re import traceback from collections import defaultdict from datetime import datetime, timedelta, timezone -from typing import Dict, Iterable, List, Optional, Set, Tuple, Type, Union, cast +from typing import Dict, Iterable, List, Optional, Set, Type, Union, cast from google.cloud import bigquery from google.cloud.bigquery.table import TableListItem @@ -57,7 +58,7 @@ ) from datahub.ingestion.source.bigquery_v2.lineage import ( BigqueryLineageExtractor, - LineageEdge, + make_lineage_edges_from_parsing_result, ) from datahub.ingestion.source.bigquery_v2.profiler import BigqueryProfiler from datahub.ingestion.source.bigquery_v2.usage import BigQueryUsageExtractor @@ -112,6 +113,7 @@ TagAssociationClass, ) from datahub.specific.dataset import DatasetPatchBuilder +from datahub.utilities.file_backed_collections import FileBackedDict from datahub.utilities.hive_schema_to_avro import ( HiveColumnToAvroConverter, get_schema_fields_for_hive_column, @@ -119,6 +121,7 @@ from datahub.utilities.mapping import Constants from datahub.utilities.perf_timer import PerfTimer from datahub.utilities.registries.domain_registry import DomainRegistry +from datahub.utilities.sqlglot_lineage import SchemaResolver, sqlglot_lineage from datahub.utilities.time import datetime_to_ts_millis logger: logging.Logger = logging.getLogger(__name__) @@ -137,6 +140,10 @@ def cleanup(config: BigQueryV2Config) -> None: os.unlink(config._credentials_path) +def _generate_sql_id(sql: str) -> str: + return hashlib.md5(sql.encode("utf-8")).hexdigest() + + @platform_name("BigQuery", doc_order=1) @config_class(BigQueryV2Config) @support_status(SupportStatus.CERTIFIED) @@ -253,8 +260,16 @@ def __init__(self, ctx: PipelineContext, config: BigQueryV2Config): # Global store of table identifiers for lineage filtering self.table_refs: Set[str] = set() - # Maps project -> view_ref -> [upstream_table_ref], for view lineage - self.view_upstream_tables: Dict[str, Dict[str, List[str]]] = defaultdict(dict) + + # We do this so that the SQL is stored in a file-backed dict, but the sql IDs are stored in memory. + # Maps project -> view_ref -> sql ID (will be used when generating lineage) + self.view_definition_ids: Dict[str, Dict[str, str]] = defaultdict(dict) + # Maps sql ID -> actual sql + self.view_definitions: FileBackedDict[str] = FileBackedDict() + + self.sql_parser_schema_resolver = SchemaResolver( + platform=self.platform, env=self.config.env + ) atexit.register(cleanup, config) @@ -274,7 +289,7 @@ def connectivity_test(client: bigquery.Client) -> CapabilityReport: return CapabilityReport(capable=True) @staticmethod - def metada_read_capability_test( + def metadata_read_capability_test( project_ids: List[str], config: BigQueryV2Config ) -> CapabilityReport: for project_id in project_ids: @@ -379,11 +394,11 @@ def test_connection(config_dict: dict) -> TestConnectionReport: if connection_conf.project_id_pattern.allowed(project.project_id): project_ids.append(project.project_id) - metada_read_capability = BigqueryV2Source.metada_read_capability_test( + metadata_read_capability = BigqueryV2Source.metadata_read_capability_test( project_ids, connection_conf ) if SourceCapability.SCHEMA_METADATA not in _report: - _report[SourceCapability.SCHEMA_METADATA] = metada_read_capability + _report[SourceCapability.SCHEMA_METADATA] = metadata_read_capability if connection_conf.include_table_lineage: lineage_capability = BigqueryV2Source.lineage_capability_test( @@ -654,19 +669,42 @@ def _process_project( def generate_lineage(self, project_id: str) -> Iterable[MetadataWorkUnit]: logger.info(f"Generate lineage for {project_id}") - lineage = self.lineage_extractor.calculate_lineage_for_project(project_id) + lineage = self.lineage_extractor.calculate_lineage_for_project( + project_id, + sql_parser_schema_resolver=self.sql_parser_schema_resolver, + ) if self.config.lineage_parse_view_ddl: - for view, upstream_tables in self.view_upstream_tables[project_id].items(): - # Override upstreams obtained by parsing audit logs as they may contain indirectly referenced tables - lineage[view] = { - LineageEdge( - table=table, - auditStamp=datetime.now(timezone.utc), - type=DatasetLineageTypeClass.VIEW, + for view, view_definition_id in self.view_definition_ids[ + project_id + ].items(): + view_definition = self.view_definitions[view_definition_id] + raw_view_lineage = sqlglot_lineage( + view_definition, + schema_resolver=self.sql_parser_schema_resolver, + default_db=project_id, + ) + if raw_view_lineage.debug_info.table_error: + logger.debug( + f"Failed to parse lineage for view {view}: {raw_view_lineage.debug_info.table_error}" ) - for table in upstream_tables - } + self.report.num_view_definitions_failed_parsing += 1 + continue + elif raw_view_lineage.debug_info.column_error: + self.report.num_view_definitions_failed_column_parsing += 1 + else: + self.report.num_view_definitions_parsed += 1 + + # For views, we override the upstreams obtained by parsing audit logs + # as they may contain indirectly referenced tables. + ts = datetime.now(timezone.utc) + lineage[view] = set( + make_lineage_edges_from_parsing_result( + raw_view_lineage, + audit_stamp=ts, + lineage_type=DatasetLineageTypeClass.VIEW, + ) + ) for lineage_key in lineage.keys(): if lineage_key not in self.table_refs: @@ -681,6 +719,7 @@ def generate_lineage(self, project_id: str) -> Iterable[MetadataWorkUnit]: lineage_info = self.lineage_extractor.get_lineage_for_table( bq_table=table_ref, + bq_table_urn=dataset_urn, platform=self.platform, lineage_metadata=lineage, ) @@ -843,14 +882,9 @@ def _process_view( ) self.table_refs.add(table_ref) if self.config.lineage_parse_view_ddl: - upstream_tables = self.lineage_extractor.parse_view_lineage( - project_id, dataset_name, view - ) - if upstream_tables is not None: - self.view_upstream_tables[project_id][table_ref] = [ - str(BigQueryTableRef(table_id).get_sanitized_table_ref()) - for table_id in upstream_tables - ] + view_definition_id = _generate_sql_id(view.view_definition) + self.view_definition_ids[project_id][table_ref] = view_definition_id + self.view_definitions[view_definition_id] = view.view_definition view.column_count = len(columns) if not view.column_count: @@ -1028,12 +1062,11 @@ def gen_dataset_workunits( def gen_lineage( self, dataset_urn: str, - lineage_info: Optional[Tuple[UpstreamLineage, Dict[str, str]]] = None, + upstream_lineage: Optional[UpstreamLineage] = None, ) -> Iterable[MetadataWorkUnit]: - if lineage_info is None: + if upstream_lineage is None: return - upstream_lineage, upstream_column_props = lineage_info if upstream_lineage is not None: if self.config.incremental_lineage: patch_builder: DatasetPatchBuilder = DatasetPatchBuilder( @@ -1050,6 +1083,9 @@ def gen_lineage( for mcp in patch_builder.build() ] else: + if not self.config.extract_column_lineage: + upstream_lineage.fineGrainedLineages = None + yield from [ MetadataChangeProposalWrapper( entityUrn=dataset_urn, aspect=upstream_lineage @@ -1148,6 +1184,12 @@ def gen_schema_metadata( # fields=[], fields=self.gen_schema_fields(columns), ) + + if self.config.lineage_parse_view_ddl or self.config.lineage_use_sql_parser: + self.sql_parser_schema_resolver.add_schema_metadata( + dataset_urn, schema_metadata + ) + return MetadataChangeProposalWrapper( entityUrn=dataset_urn, aspect=schema_metadata ).as_workunit() diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py index f1023fa6c8d6b..7287dc1b67d73 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py @@ -3,6 +3,7 @@ from datetime import timedelta from typing import Any, Dict, List, Optional +import pydantic from pydantic import Field, PositiveInt, PrivateAttr, root_validator from datahub.configuration.common import AllowDenyPattern @@ -123,7 +124,7 @@ class BigQueryV2Config( lineage_use_sql_parser: bool = Field( default=True, - description="Use sql parser to resolve view/table lineage. Only invoked on tables with both upstream tables and views. Used to distinguish between direct/base objects accessed, to only emit upstream lineage for directly accessed objects.", + description="Use sql parser to resolve view/table lineage.", ) lineage_parse_view_ddl: bool = Field( default=True, @@ -135,6 +136,22 @@ class BigQueryV2Config( description="This parameter ignores the lowercase pattern stipulated in the SQLParser. NOTE: Ignored if lineage_use_sql_parser is False.", ) + extract_column_lineage: bool = Field( + # TODO: Flip this default to True once we support patching column-level lineage. + default=False, + description="If enabled, generate column level lineage. " + "Requires lineage_use_sql_parser to be enabled. " + "This and `incremental_lineage` cannot both be enabled.", + ) + + @pydantic.validator("extract_column_lineage") + def validate_column_lineage(cls, v: bool, values: Dict[str, Any]) -> bool: + if v and values.get("incremental_lineage"): + raise ValueError( + "Cannot enable `extract_column_lineage` and `incremental_lineage` at the same time." + ) + return v + extract_lineage_from_catalog: bool = Field( default=False, description="This flag enables the data lineage extraction from Data Lineage API exposed by Google Data Catalog. NOTE: This extractor can't build views lineage. It's recommended to enable the view's DDL parsing. Read the docs to have more information about: https://cloud.google.com/data-catalog/docs/concepts/about-data-lineage", diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py index 479fc69806080..b86f5cabc6b14 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py @@ -75,6 +75,11 @@ class BigQueryV2Report(ProfilingSqlReport): num_filtered_query_events: int = 0 num_usage_query_hash_collisions: int = 0 num_operational_stats_workunits_emitted: int = 0 + + num_view_definitions_parsed: int = 0 + num_view_definitions_failed_parsing: int = 0 + num_view_definitions_failed_column_parsing: int = 0 + read_reasons_stat: Counter[str] = dataclasses.field( default_factory=collections.Counter ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py index 15c54302e83a4..2450dbd0e2391 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py @@ -319,7 +319,7 @@ class BigqueryQuery: -- We filter column limit + 1 to make sure we warn about the limit being reached but not reading too much data where column_num <= {column_limit} and shard_num = 1 ORDER BY - table_catalog, table_schema, table_name, ordinal_position, column_num ASC, table_name, data_type DESC""" + table_catalog, table_schema, table_name, ordinal_position, column_num ASC, data_type DESC""" columns_for_table: str = """ select diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py index 398e83ea23b3b..68cd9051ab5b7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py @@ -1,9 +1,10 @@ import collections +import itertools import logging import textwrap from dataclasses import dataclass from datetime import datetime, timezone -from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Union +from typing import Any, Callable, Dict, FrozenSet, Iterable, List, Optional, Set, Union import humanfriendly from google.cloud.bigquery import Client as BigQueryClient @@ -22,7 +23,6 @@ ) from datahub.ingestion.source.bigquery_v2.bigquery_config import BigQueryV2Config from datahub.ingestion.source.bigquery_v2.bigquery_report import BigQueryV2Report -from datahub.ingestion.source.bigquery_v2.bigquery_schema import BigqueryView from datahub.ingestion.source.bigquery_v2.common import ( BQ_DATE_SHARD_FORMAT, BQ_DATETIME_FORMAT, @@ -32,21 +32,146 @@ from datahub.metadata.schema_classes import ( AuditStampClass, DatasetLineageTypeClass, + FineGrainedLineageClass, + FineGrainedLineageDownstreamTypeClass, + FineGrainedLineageUpstreamTypeClass, UpstreamClass, UpstreamLineageClass, ) from datahub.utilities import memory_footprint -from datahub.utilities.bigquery_sql_parser import BigQuerySQLParser from datahub.utilities.perf_timer import PerfTimer +from datahub.utilities.sqlglot_lineage import ( + SchemaResolver, + SqlParsingResult, + sqlglot_lineage, +) +from datahub.utilities.urns.dataset_urn import DatasetUrn logger: logging.Logger = logging.getLogger(__name__) +@dataclass(order=True, eq=True, frozen=True) +class LineageEdgeColumnMapping: + out_column: str + in_columns: FrozenSet[str] + + @dataclass(order=True, eq=True, frozen=True) class LineageEdge: - table: str + table: str # in BigQueryTableRef format + column_mapping: FrozenSet[LineageEdgeColumnMapping] + auditStamp: datetime type: str = DatasetLineageTypeClass.TRANSFORMED + column_confidence: float = 0.0 + + +def _merge_lineage_edge_columns( + a: Optional[LineageEdge], b: LineageEdge +) -> LineageEdge: + if a is None: + return b + assert a.table == b.table + + merged_col_mapping: Dict[str, Set[str]] = collections.defaultdict(set) + for col_mapping in itertools.chain(a.column_mapping, b.column_mapping): + merged_col_mapping[col_mapping.out_column].update(col_mapping.in_columns) + + return LineageEdge( + table=a.table, + column_mapping=frozenset( + LineageEdgeColumnMapping( + out_column=out_column, + in_columns=frozenset(in_columns), + ) + for out_column, in_columns in merged_col_mapping.items() + ), + auditStamp=a.auditStamp, + type=a.type, + column_confidence=min(a.column_confidence, b.column_confidence), + ) + + +def _follow_column_lineage( + temp: LineageEdge, + upstream: LineageEdge, +) -> LineageEdge: + """ + Collapse lineage of `base -> temp -> upstream` into `base -> upstream`. + """ + + # This is a mapping from temp's column -> upstream's columns. + upstream_col_mapping: Dict[str, Set[str]] = collections.defaultdict(set) + for col_mapping in upstream.column_mapping: + upstream_col_mapping[col_mapping.out_column].update(col_mapping.in_columns) + + # This code can be a bit confusing. + # - temp.column_mapping[i].out_column is a column in base. + # - temp.column_mapping[i].in_columns are columns in temp. + # - For each temp_col in temp.column_mapping[i].in_columns, we want to replace it with + # the columns in upstream that it depends on. + return LineageEdge( + table=upstream.table, + column_mapping=frozenset( + LineageEdgeColumnMapping( + out_column=base_column_mapping.out_column, + in_columns=frozenset( + col + for temp_col in base_column_mapping.in_columns + for col in upstream_col_mapping.get(temp_col, set()) + ), + ) + for base_column_mapping in temp.column_mapping + ), + # We use the audit stamp and type from temp, since it's "closer" to the base. + auditStamp=temp.auditStamp, + type=temp.type, + column_confidence=min(temp.column_confidence, upstream.column_confidence), + ) + + +def make_lineage_edges_from_parsing_result( + sql_lineage: SqlParsingResult, audit_stamp: datetime, lineage_type: str +) -> List[LineageEdge]: + + # Note: This ignores the out_tables section of the sql parsing result. + audit_stamp = datetime.now(timezone.utc) + + # Invariant: table_edges[k].table == k + table_edges: Dict[str, LineageEdge] = {} + + # This isn't the most efficient code - it iterates over the same data multiple times, + # once per in_table entry. + for table_urn in sql_lineage.in_tables: + column_mapping: Dict[str, FrozenSet[str]] = {} + + for column_lineage in sql_lineage.column_lineage or []: + out_column = column_lineage.downstream.column + column_mapping[out_column] = frozenset( + upstream_column_info.column + for upstream_column_info in column_lineage.upstreams + if upstream_column_info.table == table_urn + ) + + table_name = str( + BigQueryTableRef.from_bigquery_table( + BigqueryTableIdentifier.from_string_name( + DatasetUrn.create_from_string(table_urn).get_dataset_name() + ) + ) + ) + table_edges[table_name] = LineageEdge( + table=table_name, + column_mapping=frozenset( + LineageEdgeColumnMapping(out_column=out_column, in_columns=in_columns) + for out_column, in_columns in column_mapping.items() + ), + auditStamp=audit_stamp, + type=lineage_type, + column_confidence=sql_lineage.debug_info.confidence, + ) + + return list(table_edges.values()) class BigqueryLineageExtractor: @@ -232,6 +357,7 @@ def lineage_via_catalog_lineage_api( ) ) ), + column_mapping=frozenset(), auditStamp=curr_date, ) for source_table in upstreams @@ -272,7 +398,7 @@ def _get_parsed_audit_log_events(self, project_id: str) -> Iterable[QueryEvent]: def _get_bigquery_log_entries( self, client: GCPLoggingClient, limit: Optional[int] = None - ) -> Union[Iterable[AuditLogEntry]]: + ) -> Iterable[AuditLogEntry]: self.report.num_total_log_entries[client.project] = 0 # Add a buffer to start and end time to account for delays in logging events. start_time = (self.config.start_time - self.config.max_query_duration).strftime( @@ -421,7 +547,9 @@ def _parse_exported_bigquery_audit_metadata( return event def _create_lineage_map( - self, entries: Iterable[QueryEvent] + self, + entries: Iterable[QueryEvent], + sql_parser_schema_resolver: SchemaResolver, ) -> Dict[str, Set[LineageEdge]]: logger.info("Entering create lineage map function") lineage_map: Dict[str, Set[LineageEdge]] = collections.defaultdict(set) @@ -430,139 +558,98 @@ def _create_lineage_map( self.report.num_total_lineage_entries.get(e.project_id, 0) + 1 ) - if e.destinationTable is None or not ( - e.referencedTables or e.referencedViews - ): + ts = e.end_time if e.end_time else datetime.now(tz=timezone.utc) + + # Note that we trust the audit log to have the correct destination table. + destination_table = e.destinationTable + destination_table_str = str(destination_table) + if not destination_table: self.report.num_skipped_lineage_entries_missing_data[e.project_id] += 1 continue if not self.config.dataset_pattern.allowed( - e.destinationTable.table_identifier.dataset + destination_table.table_identifier.dataset ) or not self.config.table_pattern.allowed( - e.destinationTable.table_identifier.get_table_name() + destination_table.table_identifier.get_table_name() ): self.report.num_skipped_lineage_entries_not_allowed[e.project_id] += 1 continue + # If there is a view being referenced then bigquery sends both the view as well as underlying table + # in the references. There is no distinction between direct/base objects accessed. Doing sql parsing + # ensures that we only use direct objects accessed for lineage. + # + # Because of this, we use the SQL parser as the primary source of truth for lineage + # and the audit log as a fallback. lineage_from_event: Set[LineageEdge] = set() - destination_table_str = str(e.destinationTable) - has_table = False - for ref_table in e.referencedTables: - if str(ref_table) != destination_table_str: - lineage_from_event.add( - LineageEdge( - table=str(ref_table), - auditStamp=e.end_time - if e.end_time - else datetime.now(tz=timezone.utc), - ) + # Try the sql parser first. + if self.config.lineage_use_sql_parser: + raw_lineage = sqlglot_lineage( + e.query, + schema_resolver=sql_parser_schema_resolver, + default_db=e.project_id, + ) + if raw_lineage.debug_info.table_error: + logger.debug( + f"Sql Parser failed on query: {e.query}. It won't cause any major issues, but " + f"queries referencing views may contain extra lineage for the tables underlying those views. " + f"The error was {raw_lineage.debug_info.table_error}." ) - has_table = True - has_view = False - for ref_view in e.referencedViews: - if str(ref_view) != destination_table_str: - lineage_from_event.add( - LineageEdge( - table=str(ref_view), - auditStamp=e.end_time - if e.end_time - else datetime.now(tz=timezone.utc), - ) + self.report.num_lineage_entries_sql_parser_failure[ + e.project_id + ] += 1 + + lineage_from_event = set( + make_lineage_edges_from_parsing_result( + raw_lineage, + audit_stamp=ts, + lineage_type=DatasetLineageTypeClass.TRANSFORMED, ) - has_view = True + ) if not lineage_from_event: - self.report.num_skipped_lineage_entries_other[e.project_id] += 1 - elif self.config.lineage_use_sql_parser and has_table and has_view: - # If there is a view being referenced then bigquery sends both the view as well as underlying table - # in the references. There is no distinction between direct/base objects accessed. So doing sql parsing - # to ensure we only use direct objects accessed for lineage - try: - parser = BigQuerySQLParser( - e.query, - self.config.sql_parser_use_external_process, - use_raw_names=self.config.lineage_sql_parser_use_raw_names, - ) - referenced_objs = set( - map(lambda x: x.split(".")[-1], parser.get_tables()) - ) - except Exception as ex: - logger.debug( - f"Sql Parser failed on query: {e.query}. It won't cause any issue except table/view lineage can't be detected reliably. The error was {ex}." - ) - self.report.num_lineage_entries_sql_parser_failure[ + # Fallback to the audit log if the sql parser didn't find anything. + if not (e.referencedTables or e.referencedViews): + self.report.num_skipped_lineage_entries_missing_data[ e.project_id ] += 1 continue - new_lineage = set() - for lineage in lineage_from_event: - name = lineage.table.split("/")[-1] - if name in referenced_objs: - new_lineage.add(lineage) - lineage_from_event = new_lineage + + for ref_table_or_view in [*e.referencedTables, *e.referencedViews]: + if str(ref_table_or_view) != destination_table_str: + lineage_from_event.add( + LineageEdge( + table=str(ref_table_or_view), + auditStamp=ts, + column_mapping=frozenset(), + column_confidence=0.1, + ) + ) + + if not lineage_from_event: + self.report.num_skipped_lineage_entries_other[e.project_id] += 1 + continue lineage_map[destination_table_str].update(lineage_from_event) logger.info("Exiting create lineage map function") return lineage_map - def parse_view_lineage( - self, project: str, dataset: str, view: BigqueryView - ) -> Optional[List[BigqueryTableIdentifier]]: - if not view.view_definition: - return None - - parsed_tables = set() - try: - parser = BigQuerySQLParser( - view.view_definition, - self.config.sql_parser_use_external_process, - use_raw_names=self.config.lineage_sql_parser_use_raw_names, - ) - tables = parser.get_tables() - except Exception as ex: - logger.debug( - f"View {view.name} definination sql parsing failed on query: {view.view_definition}. " - f"Edge from physical table to view won't be added. The error was {ex}." - ) - return None - - for table in tables: - parts = table.split(".") - if len(parts) == 1: - parsed_tables.add( - BigqueryTableIdentifier( - project_id=project, dataset=dataset, table=table - ) - ) - elif len(parts) == 2: - parsed_tables.add( - BigqueryTableIdentifier( - project_id=project, dataset=parts[0], table=parts[1] - ) - ) - elif len(parts) == 3: - parsed_tables.add( - BigqueryTableIdentifier( - project_id=parts[0], dataset=parts[1], table=parts[2] - ) - ) - else: - logger.warning( - f"Invalid table identifier {table} when parsing view lineage for view {view.name}" - ) - - return list(parsed_tables) - - def _compute_bigquery_lineage(self, project_id: str) -> Dict[str, Set[LineageEdge]]: + def _compute_bigquery_lineage( + self, + project_id: str, + sql_parser_schema_resolver: SchemaResolver, + ) -> Dict[str, Set[LineageEdge]]: lineage_metadata: Dict[str, Set[LineageEdge]] try: if self.config.extract_lineage_from_catalog: lineage_metadata = self.lineage_via_catalog_lineage_api(project_id) else: events = self._get_parsed_audit_log_events(project_id) - lineage_metadata = self._create_lineage_map(events) + lineage_metadata = self._create_lineage_map( + events, sql_parser_schema_resolver + ) except Exception as e: if project_id: self.report.lineage_failed_extraction.append(project_id) @@ -585,40 +672,67 @@ def get_upstream_tables( self, bq_table: BigQueryTableRef, lineage_metadata: Dict[str, Set[LineageEdge]], - tables_seen: List[str], + edges_seen: Optional[Set[LineageEdge]] = None, ) -> Set[LineageEdge]: - upstreams: Set[LineageEdge] = set() - for ref_lineage in lineage_metadata[str(bq_table)]: - ref_table = ref_lineage.table - upstream_table = BigQueryTableRef.from_string_name(ref_table) + if edges_seen is None: + edges_seen = set() + + upstreams: Dict[str, LineageEdge] = {} + for upstream_lineage in lineage_metadata[str(bq_table)]: + upstream_table_ref = upstream_lineage.table + upstream_table = BigQueryTableRef.from_string_name(upstream_table_ref) if upstream_table.is_temporary_table( [self.config.temp_table_dataset_prefix] ): # making sure we don't process a table twice and not get into a recursive loop - if ref_table in tables_seen: + if upstream_lineage in edges_seen: logger.debug( - f"Skipping table {ref_table} because it was seen already" + f"Skipping table {upstream_lineage} because it was seen already" ) continue - tables_seen.append(ref_table) - if ref_table in lineage_metadata: - upstreams = upstreams.union( - self.get_upstream_tables( - upstream_table, - lineage_metadata=lineage_metadata, - tables_seen=tables_seen, + edges_seen.add(upstream_lineage) + + if upstream_table_ref in lineage_metadata: + # `upstream_table` is a temporary table. + # We don't want it in the lineage, but we do want its upstreams. + # When following lineage for a temp table, we need to merge the column lineage. + + for temp_table_upstream in self.get_upstream_tables( + upstream_table, + lineage_metadata=lineage_metadata, + edges_seen=edges_seen, + ): + ref_temp_table_upstream = temp_table_upstream.table + + # Replace `bq_table -> upstream_table -> temp_table_upstream` + # with `bq_table -> temp_table_upstream`, merging the column lineage. + collapsed_lineage = _follow_column_lineage( + upstream_lineage, temp_table_upstream + ) + + upstreams[ + ref_temp_table_upstream + ] = _merge_lineage_edge_columns( + upstreams.get(ref_temp_table_upstream), + collapsed_lineage, ) - ) else: - upstreams.add(ref_lineage) + upstreams[upstream_table_ref] = _merge_lineage_edge_columns( + upstreams.get(upstream_table_ref), + upstream_lineage, + ) - return upstreams + return set(upstreams.values()) def calculate_lineage_for_project( - self, project_id: str + self, + project_id: str, + sql_parser_schema_resolver: SchemaResolver, ) -> Dict[str, Set[LineageEdge]]: with PerfTimer() as timer: - lineage = self._compute_bigquery_lineage(project_id) + lineage = self._compute_bigquery_lineage( + project_id, sql_parser_schema_resolver + ) self.report.lineage_extraction_sec[project_id] = round( timer.elapsed_seconds(), 2 @@ -629,23 +743,26 @@ def calculate_lineage_for_project( def get_lineage_for_table( self, bq_table: BigQueryTableRef, + bq_table_urn: str, lineage_metadata: Dict[str, Set[LineageEdge]], platform: str, - ) -> Optional[Tuple[UpstreamLineageClass, Dict[str, str]]]: + ) -> Optional[UpstreamLineageClass]: upstream_list: List[UpstreamClass] = [] + fine_grained_lineages: List[FineGrainedLineageClass] = [] # Sorting the list of upstream lineage events in order to avoid creating multiple aspects in backend # even if the lineage is same but the order is different. - for upstream in sorted( - self.get_upstream_tables(bq_table, lineage_metadata, tables_seen=[]) - ): + for upstream in sorted(self.get_upstream_tables(bq_table, lineage_metadata)): upstream_table = BigQueryTableRef.from_string_name(upstream.table) + upstream_table_urn = mce_builder.make_dataset_urn_with_platform_instance( + platform, + upstream_table.table_identifier.get_table_name(), + self.config.platform_instance, + self.config.env, + ) + + # Generate table-level lineage. upstream_table_class = UpstreamClass( - dataset=mce_builder.make_dataset_urn_with_platform_instance( - platform, - upstream_table.table_identifier.get_table_name(), - self.config.platform_instance, - self.config.env, - ), + dataset=upstream_table_urn, type=upstream.type, auditStamp=AuditStampClass( actor="urn:li:corpuser:datahub", @@ -658,11 +775,35 @@ def get_lineage_for_table( ) current_lineage_map.add(str(upstream_table)) self.report.upstream_lineage[str(bq_table)] = current_lineage_map + upstream_list.append(upstream_table_class) + # Generate column-level lineage. + for col_lineage_edge in upstream.column_mapping: + fine_grained_lineage = FineGrainedLineageClass( + downstreamType=FineGrainedLineageDownstreamTypeClass.FIELD, + downstreams=[ + mce_builder.make_schema_field_urn( + bq_table_urn, col_lineage_edge.out_column + ) + ], + upstreamType=FineGrainedLineageUpstreamTypeClass.FIELD_SET, + upstreams=[ + mce_builder.make_schema_field_urn( + upstream_table_urn, upstream_col + ) + for upstream_col in col_lineage_edge.in_columns + ], + confidenceScore=upstream.column_confidence, + ) + fine_grained_lineages.append(fine_grained_lineage) + if upstream_list: - upstream_lineage = UpstreamLineageClass(upstreams=upstream_list) - return upstream_lineage, {} + upstream_lineage = UpstreamLineageClass( + upstreams=upstream_list, + fineGrainedLineages=fine_grained_lineages, + ) + return upstream_lineage return None diff --git a/metadata-ingestion/src/datahub/ingestion/source_report/sql/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source_report/sql/bigquery.py deleted file mode 100644 index 9c1560186ffb6..0000000000000 --- a/metadata-ingestion/src/datahub/ingestion/source_report/sql/bigquery.py +++ /dev/null @@ -1,39 +0,0 @@ -from dataclasses import dataclass, field -from datetime import datetime -from typing import Dict, List, Optional - -import pydantic - -from datahub.ingestion.source.sql.sql_common import SQLSourceReport - - -@dataclass -class BigQueryReport(SQLSourceReport): - num_total_lineage_entries: Optional[int] = None - num_skipped_lineage_entries_missing_data: Optional[int] = None - num_skipped_lineage_entries_not_allowed: Optional[int] = None - num_skipped_lineage_entries_sql_parser_failure: Optional[int] = None - num_skipped_lineage_entries_other: Optional[int] = None - num_total_log_entries: Optional[int] = None - num_parsed_log_entires: Optional[int] = None - num_total_audit_entries: Optional[int] = None - num_parsed_audit_entires: Optional[int] = None - bigquery_audit_metadata_datasets_missing: Optional[bool] = None - lineage_metadata_entries: Optional[int] = None - include_table_lineage: Optional[bool] = None - use_date_sharded_audit_log_tables: Optional[bool] = None - log_page_size: Optional[pydantic.PositiveInt] = None - use_v2_audit_metadata: Optional[bool] = None - use_exported_bigquery_audit_metadata: Optional[bool] = None - window_start_time: Optional[datetime] = None - window_end_time: Optional[datetime] = None - log_entry_start_time: Optional[str] = None - log_entry_end_time: Optional[str] = None - audit_start_time: Optional[str] = None - audit_end_time: Optional[str] = None - upstream_lineage: Dict = field(default_factory=dict) - partition_info: Dict[str, str] = field(default_factory=dict) - table_metadata: Dict[str, List[str]] = field(default_factory=dict) - profile_table_selection_criteria: Dict[str, str] = field(default_factory=dict) - selected_profile_tables: Dict[str, List[str]] = field(default_factory=dict) - invalid_partition_ids: Dict[str, str] = field(default_factory=dict) diff --git a/metadata-ingestion/src/datahub/ingestion/source_report/usage/bigquery_usage.py b/metadata-ingestion/src/datahub/ingestion/source_report/usage/bigquery_usage.py deleted file mode 100644 index 194183895665d..0000000000000 --- a/metadata-ingestion/src/datahub/ingestion/source_report/usage/bigquery_usage.py +++ /dev/null @@ -1,36 +0,0 @@ -import collections -import dataclasses -from datetime import datetime -from typing import Counter, Optional - -from datahub.ingestion.api.source import SourceReport - - -@dataclasses.dataclass -class BigQueryUsageSourceReport(SourceReport): - dropped_table: Counter[str] = dataclasses.field(default_factory=collections.Counter) - total_log_entries: Optional[int] = None - num_read_events: Optional[int] = None - num_filtered_read_events: Optional[int] = None - num_filtered_query_events: Optional[int] = None - num_query_events: Optional[int] = None - use_v2_audit_metadata: Optional[bool] = None - log_page_size: Optional[int] = None - query_log_delay: Optional[int] = None - window_start_time: Optional[datetime] = None - window_end_time: Optional[datetime] = None - allow_pattern: Optional[str] = None - deny_pattern: Optional[str] = None - log_entry_start_time: Optional[str] = None - log_entry_end_time: Optional[str] = None - num_usage_workunits_emitted: Optional[int] = None - num_operational_stats_workunits_emitted: Optional[int] = None - read_reasons_stat: Counter[str] = dataclasses.field( - default_factory=collections.Counter - ) - operation_types_stat: Counter[str] = dataclasses.field( - default_factory=collections.Counter - ) - - def report_dropped(self, key: str) -> None: - self.dropped_table[key] += 1 diff --git a/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py b/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py index b73af0478a1da..8516a7054a9cd 100644 --- a/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py +++ b/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py @@ -22,7 +22,6 @@ def assert_sql_result_with_resolver( sql: str, *, - dialect: str, expected_file: pathlib.Path, schema_resolver: SchemaResolver, **kwargs: Any, @@ -33,7 +32,6 @@ def assert_sql_result_with_resolver( res = sqlglot_lineage( sql, - platform=dialect, schema_resolver=schema_resolver, **kwargs, ) @@ -83,7 +81,6 @@ def assert_sql_result( assert_sql_result_with_resolver( sql, - dialect=dialect, expected_file=expected_file, schema_resolver=schema_resolver, **kwargs, diff --git a/metadata-ingestion/src/datahub/utilities/file_backed_collections.py b/metadata-ingestion/src/datahub/utilities/file_backed_collections.py index 74e97e5104105..c04d2138bc116 100644 --- a/metadata-ingestion/src/datahub/utilities/file_backed_collections.py +++ b/metadata-ingestion/src/datahub/utilities/file_backed_collections.py @@ -56,29 +56,19 @@ class ConnectionWrapper: conn: sqlite3.Connection filename: pathlib.Path - _directory: Optional[tempfile.TemporaryDirectory] - _allow_table_name_reuse: bool + _temp_directory: Optional[tempfile.TemporaryDirectory] def __init__(self, filename: Optional[pathlib.Path] = None): - self._directory = None - - # In the normal case, we do not use "IF NOT EXISTS" in our create table statements - # because creating the same table twice indicates a client usage error. - # However, if you're trying to persist a file-backed dict across multiple runs, - # which happens when filename is passed explicitly, then we need to allow table name reuse. - allow_table_name_reuse = False + self._temp_directory = None # Warning: If filename is provided, the file will not be automatically cleaned up. - if filename: - allow_table_name_reuse = True - else: - self._directory = tempfile.TemporaryDirectory() - filename = pathlib.Path(self._directory.name) / _DEFAULT_FILE_NAME + if not filename: + self._temp_directory = tempfile.TemporaryDirectory() + filename = pathlib.Path(self._temp_directory.name) / _DEFAULT_FILE_NAME self.conn = sqlite3.connect(filename, isolation_level=None) self.conn.row_factory = sqlite3.Row self.filename = filename - self._allow_table_name_reuse = allow_table_name_reuse # These settings are optimized for performance. # See https://www.sqlite.org/pragma.html for more information. @@ -89,22 +79,29 @@ def __init__(self, filename: Optional[pathlib.Path] = None): self.conn.execute('PRAGMA journal_mode = "MEMORY"') self.conn.execute(f"PRAGMA journal_size_limit = {100 * 1024 * 1024}") # 100MB + @property + def allow_table_name_reuse(self) -> bool: + # In the normal case, we do not use "IF NOT EXISTS" in our create table statements + # because creating the same table twice indicates a client usage error. + # However, if you're trying to persist a file-backed dict across multiple runs, + # which happens when filename is passed explicitly, then we need to allow table name reuse. + + return self._temp_directory is None + def execute( self, sql: str, parameters: Union[Dict[str, Any], Sequence[Any]] = () ) -> sqlite3.Cursor: - # logger.debug(f"Executing <{sql}> ({parameters})") return self.conn.execute(sql, parameters) def executemany( self, sql: str, parameters: Union[Dict[str, Any], Sequence[Any]] = () ) -> sqlite3.Cursor: - # logger.debug(f"Executing many <{sql}> ({parameters})") return self.conn.executemany(sql, parameters) def close(self) -> None: self.conn.close() - if self._directory: - self._directory.cleanup() + if self._temp_directory: + self._temp_directory.cleanup() def __enter__(self) -> "ConnectionWrapper": return self @@ -197,7 +194,7 @@ def __post_init__(self) -> None: self._active_object_cache = collections.OrderedDict() # Create the table. - if_not_exists = "IF NOT EXISTS" if self._conn._allow_table_name_reuse else "" + if_not_exists = "IF NOT EXISTS" if self._conn.allow_table_name_reuse else "" self._conn.execute( f"""CREATE TABLE {if_not_exists} {self.tablename} ( key TEXT PRIMARY KEY, diff --git a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py index 189c5bb269f67..bae97b96c4544 100644 --- a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py +++ b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py @@ -46,6 +46,7 @@ class QueryType(enum.Enum): def get_query_type_of_sql(expression: sqlglot.exp.Expression) -> QueryType: + # UPGRADE: Once we use Python 3.10, replace this with a match expression. mapping = { sqlglot.exp.Create: QueryType.CREATE, sqlglot.exp.Select: QueryType.SELECT, @@ -102,7 +103,6 @@ def qualified( def from_sqlglot_table( cls, table: sqlglot.exp.Table, - dialect: str, default_db: Optional[str] = None, default_schema: Optional[str] = None, ) -> "_TableName": @@ -149,12 +149,17 @@ class ColumnLineageInfo(BaseModel): class SqlParsingDebugInfo(BaseModel, arbitrary_types_allowed=True): - confidence: float + confidence: float = 0.0 + + tables_discovered: int = 0 + table_schemas_resolved: int = 0 - tables_discovered: int - table_schemas_resolved: int + table_error: Optional[Exception] = None + column_error: Optional[Exception] = None - column_error: Optional[Exception] + @property + def error(self) -> Optional[Exception]: + return self.table_error or self.column_error class SqlParsingResult(BaseModel): @@ -169,12 +174,7 @@ class SqlParsingResult(BaseModel): # TODO include list of referenced columns debug_info: SqlParsingDebugInfo = pydantic.Field( - default_factory=lambda: SqlParsingDebugInfo( - confidence=0, - tables_discovered=0, - table_schemas_resolved=0, - column_error=None, - ), + default_factory=lambda: SqlParsingDebugInfo(), exclude=True, ) @@ -190,12 +190,9 @@ def _table_level_lineage( statement: sqlglot.Expression, dialect: str, ) -> Tuple[Set[_TableName], Set[_TableName]]: - def _raw_table_name(table: sqlglot.exp.Table) -> _TableName: - return _TableName.from_sqlglot_table(table, dialect=dialect) - # Generate table-level lineage. modified = { - _raw_table_name(expr.this) + _TableName.from_sqlglot_table(expr.this) for expr in statement.find_all( sqlglot.exp.Create, sqlglot.exp.Insert, @@ -209,7 +206,10 @@ def _raw_table_name(table: sqlglot.exp.Table) -> _TableName: } tables = ( - {_raw_table_name(table) for table in statement.find_all(sqlglot.exp.Table)} + { + _TableName.from_sqlglot_table(table) + for table in statement.find_all(sqlglot.exp.Table) + } # ignore references created in this query - modified # ignore CTEs created in this statement @@ -259,7 +259,6 @@ def get_urn_for_table(self, table: _TableName, lower: bool = False) -> str: if self.platform == "bigquery": # Normalize shard numbers and other BigQuery weirdness. - # TODO check that this is the right way to do it with contextlib.suppress(IndexError): table_name = BigqueryTableIdentifier.from_string_name( table_name @@ -513,9 +512,7 @@ def _sqlglot_force_column_normalizer( pass elif isinstance(node.expression, sqlglot.exp.Table): - table_ref = _TableName.from_sqlglot_table( - node.expression, dialect=dialect - ) + table_ref = _TableName.from_sqlglot_table(node.expression) # Parse the column name out of the node name. # Sqlglot calls .sql(), so we have to do the inverse. @@ -622,15 +619,15 @@ def _translate_internal_column_lineage( ) -def sqlglot_lineage( +def _sqlglot_lineage_inner( sql: str, - platform: str, schema_resolver: SchemaResolver, default_db: Optional[str] = None, default_schema: Optional[str] = None, ) -> SqlParsingResult: # TODO: convert datahub platform names to sqlglot dialect - dialect = platform + # TODO: Pull the platform name from the schema resolver? + dialect = schema_resolver.platform if dialect == "snowflake": # in snowflake, table identifiers must be uppercased to match sqlglot's behavior. @@ -752,3 +749,27 @@ def sqlglot_lineage( column_lineage=column_lineage_urns, debug_info=debug_info, ) + + +def sqlglot_lineage( + sql: str, + schema_resolver: SchemaResolver, + default_db: Optional[str] = None, + default_schema: Optional[str] = None, +) -> SqlParsingResult: + try: + return _sqlglot_lineage_inner( + sql=sql, + schema_resolver=schema_resolver, + default_db=default_db, + default_schema=default_schema, + ) + except Exception as e: + return SqlParsingResult( + in_tables=[], + out_tables=[], + column_lineage=None, + debug_info=SqlParsingDebugInfo( + table_error=e, + ), + ) diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_star_with_replace.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_star_with_replace.json new file mode 100644 index 0000000000000..17a801a63e3ff --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_star_with_replace.json @@ -0,0 +1,47 @@ +{ + "query_type": "CREATE", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-project2.my-dataset2.test_physical_table,PROD)" + ], + "out_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-project.my-dataset.test_table,PROD)" + ], + "column_lineage": [ + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-project.my-dataset.test_table,PROD)", + "column": "col1" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-project2.my-dataset2.test_physical_table,PROD)", + "column": "col1" + } + ] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-project.my-dataset.test_table,PROD)", + "column": "col2" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-project2.my-dataset2.test_physical_table,PROD)", + "column": "col2" + } + ] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-project.my-dataset.test_table,PROD)", + "column": "something" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-project2.my-dataset2.test_physical_table,PROD)", + "column": "something" + } + ] + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_view_from_union.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_view_from_union.json new file mode 100644 index 0000000000000..fd8a586ac74ac --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_bigquery_view_from_union.json @@ -0,0 +1,44 @@ +{ + "query_type": "CREATE", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project_2.my_dataset_2.sometable,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project_2.my_dataset_2.sometable2,PROD)" + ], + "out_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,my_view,PROD)" + ], + "column_lineage": [ + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my_view,PROD)", + "column": "col1" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project_2.my_dataset_2.sometable,PROD)", + "column": "col1" + }, + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project_2.my_dataset_2.sometable2,PROD)", + "column": "col1" + } + ] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my_view,PROD)", + "column": "col2" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project_2.my_dataset_2.sometable,PROD)", + "column": "col2" + }, + { + "table": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project_2.my_dataset_2.sometable2,PROD)", + "column": "col2" + } + ] + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py index b7ef62ba82296..0d8471291a505 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py +++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py @@ -421,6 +421,51 @@ def test_bigquery_from_sharded_table_wildcard(): ) +def test_bigquery_star_with_replace(): + assert_sql_result( + """ +CREATE VIEW `my-project.my-dataset.test_table` AS +SELECT + * REPLACE( + LOWER(something) AS something) +FROM + `my-project2.my-dataset2.test_physical_table`; +""", + dialect="bigquery", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,my-project2.my-dataset2.test_physical_table,PROD)": { + "col1": "STRING", + "col2": "STRING", + "something": "STRING", + }, + }, + expected_file=RESOURCE_DIR / "test_bigquery_star_with_replace.json", + ) + + +def test_bigquery_view_from_union(): + assert_sql_result( + """ +CREATE VIEW my_view as +select * from my_project_2.my_dataset_2.sometable +union +select * from my_project_2.my_dataset_2.sometable2 as a +""", + dialect="bigquery", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project_2.my_dataset_2.sometable,PROD)": { + "col1": "STRING", + "col2": "STRING", + }, + "urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project_2.my_dataset_2.sometable2,PROD)": { + "col1": "STRING", + "col2": "STRING", + }, + }, + expected_file=RESOURCE_DIR / "test_bigquery_view_from_union.json", + ) + + def test_snowflake_default_normalization(): assert_sql_result( """ diff --git a/metadata-ingestion/tests/unit/test_bigquery_lineage.py b/metadata-ingestion/tests/unit/test_bigquery_lineage.py index 0cc362c6695eb..c9308fd89ef72 100644 --- a/metadata-ingestion/tests/unit/test_bigquery_lineage.py +++ b/metadata-ingestion/tests/unit/test_bigquery_lineage.py @@ -7,97 +7,11 @@ ) from datahub.ingestion.source.bigquery_v2.bigquery_config import BigQueryV2Config from datahub.ingestion.source.bigquery_v2.bigquery_report import BigQueryV2Report -from datahub.ingestion.source.bigquery_v2.bigquery_schema import BigqueryView from datahub.ingestion.source.bigquery_v2.lineage import ( BigqueryLineageExtractor, LineageEdge, ) - - -def test_parse_view_lineage(): - config = BigQueryV2Config() - report = BigQueryV2Report() - extractor = BigqueryLineageExtractor(config, report) - - # ddl = "select * from some_dataset.sometable as a" - ddl = """CREATE VIEW `my-project.my-dataset.test_table` -AS SELECT - * REPLACE( - myrandom(something) AS something) -FROM - `my-project2.my-dataset2.test_physical_table`; -""" - view = BigqueryView( - name="test", - created=datetime.datetime.now(tz=datetime.timezone.utc), - last_altered=datetime.datetime.now(tz=datetime.timezone.utc), - comment="", - view_definition=ddl, - ) - tables = extractor.parse_view_lineage("my_project", "my_dataset", view) - assert tables is not None - assert 1 == len(tables) - assert "my-project2.my-dataset2.test_physical_table" == tables[0].get_table_name() - - -def test_parse_view_lineage_with_two_part_table_name(): - config = BigQueryV2Config() - report = BigQueryV2Report() - extractor = BigqueryLineageExtractor(config, report) - - ddl = "CREATE VIEW my_view as select * from some_dataset.sometable as a" - view = BigqueryView( - name="test", - created=datetime.datetime.now(tz=datetime.timezone.utc), - last_altered=datetime.datetime.now(tz=datetime.timezone.utc), - comment="", - view_definition=ddl, - ) - tables = extractor.parse_view_lineage("my_project", "my_dataset", view) - assert tables is not None - assert 1 == len(tables) - assert "my_project.some_dataset.sometable" == tables[0].get_table_name() - - -def test_one_part_table(): - config = BigQueryV2Config() - report = BigQueryV2Report() - extractor = BigqueryLineageExtractor(config, report) - - ddl = "CREATE VIEW my_view as select * from sometable as a" - view = BigqueryView( - name="test", - created=datetime.datetime.now(tz=datetime.timezone.utc), - last_altered=datetime.datetime.now(tz=datetime.timezone.utc), - comment="", - view_definition=ddl, - ) - tables = extractor.parse_view_lineage("my_project", "my_dataset", view) - assert tables is not None - assert 1 == len(tables) - assert "my_project.my_dataset.sometable" == tables[0].get_table_name() - - -def test_create_statement_with_multiple_table(): - config = BigQueryV2Config() - report = BigQueryV2Report() - extractor = BigqueryLineageExtractor(config, report) - - ddl = "CREATE VIEW my_view as select * from my_project_2.my_dataset_2.sometable union select * from my_project_2.my_dataset_2.sometable2 as a" - view = BigqueryView( - name="test", - created=datetime.datetime.now(), - last_altered=datetime.datetime.now(), - comment="", - view_definition=ddl, - ) - tables = extractor.parse_view_lineage("my_project", "my_dataset", view) - assert tables is not None - - tables.sort(key=lambda e: e.get_table_name()) - assert 2 == len(tables) - assert "my_project_2.my_dataset_2.sometable" == tables[0].get_table_name() - assert "my_project_2.my_dataset_2.sometable2" == tables[1].get_table_name() +from datahub.utilities.sqlglot_lineage import SchemaResolver def test_lineage_with_timestamps(): @@ -164,13 +78,15 @@ def test_lineage_with_timestamps(): ) lineage_map: Dict[str, Set[LineageEdge]] = extractor._create_lineage_map( - iter(lineage_entries) + iter(lineage_entries), + sql_parser_schema_resolver=SchemaResolver(platform="bigquery"), ) upstream_lineage = extractor.get_lineage_for_table( bq_table=bq_table, + bq_table_urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project.my_dataset.my_table,PROD)", lineage_metadata=lineage_map, platform="bigquery", ) assert upstream_lineage - assert len(upstream_lineage[0].upstreams) == 4 + assert len(upstream_lineage.upstreams) == 4 diff --git a/metadata-ingestion/tests/unit/test_bigquery_source.py b/metadata-ingestion/tests/unit/test_bigquery_source.py index 3efca8d088eed..fc8ca166b105a 100644 --- a/metadata-ingestion/tests/unit/test_bigquery_source.py +++ b/metadata-ingestion/tests/unit/test_bigquery_source.py @@ -23,7 +23,10 @@ BigqueryProject, BigqueryView, ) -from datahub.ingestion.source.bigquery_v2.lineage import LineageEdge +from datahub.ingestion.source.bigquery_v2.lineage import ( + LineageEdge, + LineageEdgeColumnMapping, +) from datahub.metadata.com.linkedin.pegasus2avro.dataset import ViewProperties from datahub.metadata.schema_classes import MetadataChangeProposalClass @@ -262,8 +265,14 @@ def test_simple_upstream_table_generation(): } ) source = BigqueryV2Source(config=config, ctx=PipelineContext(run_id="test")) - lineage_metadata = {str(a): {LineageEdge(table=str(b), auditStamp=datetime.now())}} - upstreams = source.lineage_extractor.get_upstream_tables(a, lineage_metadata, []) + lineage_metadata = { + str(a): { + LineageEdge( + table=str(b), auditStamp=datetime.now(), column_mapping=frozenset() + ) + } + } + upstreams = source.lineage_extractor.get_upstream_tables(a, lineage_metadata) assert len(upstreams) == 1 assert list(upstreams)[0].table == str(b) @@ -288,12 +297,18 @@ def test_upstream_table_generation_with_temporary_table_without_temp_upstream(): ) source = BigqueryV2Source(config=config, ctx=PipelineContext(run_id="test")) - lineage_metadata = {str(a): {LineageEdge(table=str(b), auditStamp=datetime.now())}} - upstreams = source.lineage_extractor.get_upstream_tables(a, lineage_metadata, []) + lineage_metadata = { + str(a): { + LineageEdge( + table=str(b), auditStamp=datetime.now(), column_mapping=frozenset() + ) + } + } + upstreams = source.lineage_extractor.get_upstream_tables(a, lineage_metadata) assert list(upstreams) == [] -def test_upstream_table_generation_with_temporary_table_with_temp_upstream(): +def test_upstream_table_column_lineage_with_temp_table(): from datahub.ingestion.api.common import PipelineContext a: BigQueryTableRef = BigQueryTableRef( @@ -320,12 +335,51 @@ def test_upstream_table_generation_with_temporary_table_with_temp_upstream(): source = BigqueryV2Source(config=config, ctx=PipelineContext(run_id="test")) lineage_metadata = { - str(a): {LineageEdge(table=str(b), auditStamp=datetime.now())}, - str(b): {LineageEdge(table=str(c), auditStamp=datetime.now())}, + str(a): { + LineageEdge( + table=str(b), + auditStamp=datetime.now(), + column_mapping=frozenset( + [ + LineageEdgeColumnMapping( + "a_col1", in_columns=frozenset(["b_col2", "b_col3"]) + ) + ] + ), + column_confidence=0.8, + ) + }, + str(b): { + LineageEdge( + table=str(c), + auditStamp=datetime.now(), + column_mapping=frozenset( + [ + LineageEdgeColumnMapping( + "b_col2", in_columns=frozenset(["c_col1", "c_col2"]) + ), + LineageEdgeColumnMapping( + "b_col3", in_columns=frozenset(["c_col2", "c_col3"]) + ), + ] + ), + column_confidence=0.7, + ) + }, } - upstreams = source.lineage_extractor.get_upstream_tables(a, lineage_metadata, []) + upstreams = source.lineage_extractor.get_upstream_tables(a, lineage_metadata) assert len(upstreams) == 1 - assert list(upstreams)[0].table == str(c) + + upstream = list(upstreams)[0] + assert upstream.table == str(c) + assert upstream.column_mapping == frozenset( + [ + LineageEdgeColumnMapping( + "a_col1", in_columns=frozenset(["c_col1", "c_col2", "c_col3"]) + ) + ] + ) + assert upstream.column_confidence == 0.7 def test_upstream_table_generation_with_temporary_table_with_multiple_temp_upstream(): @@ -362,14 +416,26 @@ def test_upstream_table_generation_with_temporary_table_with_multiple_temp_upstr ) source = BigqueryV2Source(config=config, ctx=PipelineContext(run_id="test")) lineage_metadata = { - str(a): {LineageEdge(table=str(b), auditStamp=datetime.now())}, + str(a): { + LineageEdge( + table=str(b), auditStamp=datetime.now(), column_mapping=frozenset() + ) + }, str(b): { - LineageEdge(table=str(c), auditStamp=datetime.now()), - LineageEdge(table=str(d), auditStamp=datetime.now()), + LineageEdge( + table=str(c), auditStamp=datetime.now(), column_mapping=frozenset() + ), + LineageEdge( + table=str(d), auditStamp=datetime.now(), column_mapping=frozenset() + ), + }, + str(d): { + LineageEdge( + table=str(e), auditStamp=datetime.now(), column_mapping=frozenset() + ) }, - str(d): {LineageEdge(table=str(e), auditStamp=datetime.now())}, } - upstreams = source.lineage_extractor.get_upstream_tables(a, lineage_metadata, []) + upstreams = source.lineage_extractor.get_upstream_tables(a, lineage_metadata) sorted_list = list(upstreams) sorted_list.sort() assert sorted_list[0].table == str(c) From 75b79e160afddb73d08ed74ee49e3ea5966d6a5e Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Tue, 11 Jul 2023 16:40:20 -0400 Subject: [PATCH 123/222] build(ingest): Inline import testing utils for check cli (#8400) --- metadata-ingestion/src/datahub/cli/check_cli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/cli/check_cli.py b/metadata-ingestion/src/datahub/cli/check_cli.py index 5cee21e45b97a..bec1672264b88 100644 --- a/metadata-ingestion/src/datahub/cli/check_cli.py +++ b/metadata-ingestion/src/datahub/cli/check_cli.py @@ -15,8 +15,6 @@ from datahub.ingestion.source.source_registry import source_registry from datahub.ingestion.transformer.transform_registry import transform_registry from datahub.telemetry import telemetry -from datahub.testing.compare_metadata_json import diff_metadata_json, load_json_file -from datahub.testing.mcp_diff import MCPDiff logger = logging.getLogger(__name__) @@ -101,8 +99,12 @@ def metadata_diff( ) -> None: """Compare two metadata (MCE or MCP) JSON files. + To use this command, you must install the acryl-datahub[testing-utils] extra. + Comparison is more sophisticated for files composed solely of MCPs. """ + from datahub.testing.compare_metadata_json import diff_metadata_json, load_json_file + from datahub.testing.mcp_diff import MCPDiff actual = load_json_file(actual_file) expected = load_json_file(expected_file) From 779d692cdc7a74ff85f33960ff7572521736329b Mon Sep 17 00:00:00 2001 From: sudhakara Date: Wed, 12 Jul 2023 02:39:54 +0530 Subject: [PATCH 124/222] refactor(ui): uniform ordering of items on the entities sidebar section (#8365) Co-authored-by: sudhakara.st Co-authored-by: Chris Collins Co-authored-by: Joshua Eilers --- datahub-web-react/src/app/entity/chart/ChartEntity.tsx | 6 +++--- .../src/app/entity/container/ContainerEntity.tsx | 6 +++--- .../src/app/entity/dashboard/DashboardEntity.tsx | 10 +++++----- .../src/app/entity/dataFlow/DataFlowEntity.tsx | 10 +++++----- .../src/app/entity/dataJob/DataJobEntity.tsx | 10 +++++----- .../src/app/entity/mlFeature/MLFeatureEntity.tsx | 10 +++++----- .../app/entity/mlFeatureTable/MLFeatureTableEntity.tsx | 10 +++++----- .../src/app/entity/mlModel/MLModelEntity.tsx | 10 +++++----- .../src/app/entity/mlModelGroup/MLModelGroupEntity.tsx | 10 +++++----- .../src/app/entity/mlPrimaryKey/MLPrimaryKeyEntity.tsx | 10 +++++----- 10 files changed, 46 insertions(+), 46 deletions(-) diff --git a/datahub-web-react/src/app/entity/chart/ChartEntity.tsx b/datahub-web-react/src/app/entity/chart/ChartEntity.tsx index aaf4d4a9689f3..90032285cd35b 100644 --- a/datahub-web-react/src/app/entity/chart/ChartEntity.tsx +++ b/datahub-web-react/src/app/entity/chart/ChartEntity.tsx @@ -130,6 +130,9 @@ export class ChartEntity implements Entity { { component: SidebarAboutSection, }, + { + component: SidebarOwnerSection, + }, { component: SidebarTagsSection, properties: { @@ -137,9 +140,6 @@ export class ChartEntity implements Entity { hasTerms: true, }, }, - { - component: SidebarOwnerSection, - }, { component: SidebarDomainSection, }, diff --git a/datahub-web-react/src/app/entity/container/ContainerEntity.tsx b/datahub-web-react/src/app/entity/container/ContainerEntity.tsx index d48ef8f1db595..201dcb9e4487a 100644 --- a/datahub-web-react/src/app/entity/container/ContainerEntity.tsx +++ b/datahub-web-react/src/app/entity/container/ContainerEntity.tsx @@ -88,6 +88,9 @@ export class ContainerEntity implements Entity { { component: SidebarAboutSection, }, + { + component: SidebarOwnerSection, + }, { component: SidebarTagsSection, properties: { @@ -95,9 +98,6 @@ export class ContainerEntity implements Entity { hasTerms: true, }, }, - { - component: SidebarOwnerSection, - }, { component: SidebarDomainSection, }, diff --git a/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx b/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx index b8dc6861711ec..d948b21a46262 100644 --- a/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx +++ b/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx @@ -139,16 +139,16 @@ export class DashboardEntity implements Entity { component: SidebarAboutSection, }, { - component: SidebarTagsSection, + component: SidebarOwnerSection, properties: { - hasTags: true, - hasTerms: true, + defaultOwnerType: OwnershipType.TechnicalOwner, }, }, { - component: SidebarOwnerSection, + component: SidebarTagsSection, properties: { - defaultOwnerType: OwnershipType.TechnicalOwner, + hasTags: true, + hasTerms: true, }, }, { diff --git a/datahub-web-react/src/app/entity/dataFlow/DataFlowEntity.tsx b/datahub-web-react/src/app/entity/dataFlow/DataFlowEntity.tsx index 3f2b1a6dfd273..c6f7c8b6a6cf7 100644 --- a/datahub-web-react/src/app/entity/dataFlow/DataFlowEntity.tsx +++ b/datahub-web-react/src/app/entity/dataFlow/DataFlowEntity.tsx @@ -85,16 +85,16 @@ export class DataFlowEntity implements Entity { component: SidebarAboutSection, }, { - component: SidebarTagsSection, + component: SidebarOwnerSection, properties: { - hasTags: true, - hasTerms: true, + defaultOwnerType: OwnershipType.TechnicalOwner, }, }, { - component: SidebarOwnerSection, + component: SidebarTagsSection, properties: { - defaultOwnerType: OwnershipType.TechnicalOwner, + hasTags: true, + hasTerms: true, }, }, { diff --git a/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx b/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx index 398e79688a2ca..a2a369ec53ecf 100644 --- a/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx +++ b/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx @@ -108,16 +108,16 @@ export class DataJobEntity implements Entity { component: SidebarAboutSection, }, { - component: SidebarTagsSection, + component: SidebarOwnerSection, properties: { - hasTags: true, - hasTerms: true, + defaultOwnerType: OwnershipType.TechnicalOwner, }, }, { - component: SidebarOwnerSection, + component: SidebarTagsSection, properties: { - defaultOwnerType: OwnershipType.TechnicalOwner, + hasTags: true, + hasTerms: true, }, }, { diff --git a/datahub-web-react/src/app/entity/mlFeature/MLFeatureEntity.tsx b/datahub-web-react/src/app/entity/mlFeature/MLFeatureEntity.tsx index 4e6ea985d76dc..8fddae7c15186 100644 --- a/datahub-web-react/src/app/entity/mlFeature/MLFeatureEntity.tsx +++ b/datahub-web-react/src/app/entity/mlFeature/MLFeatureEntity.tsx @@ -91,16 +91,16 @@ export class MLFeatureEntity implements Entity { component: SidebarAboutSection, }, { - component: SidebarTagsSection, + component: SidebarOwnerSection, properties: { - hasTags: true, - hasTerms: true, + defaultOwnerType: OwnershipType.TechnicalOwner, }, }, { - component: SidebarOwnerSection, + component: SidebarTagsSection, properties: { - defaultOwnerType: OwnershipType.TechnicalOwner, + hasTags: true, + hasTerms: true, }, }, { diff --git a/datahub-web-react/src/app/entity/mlFeatureTable/MLFeatureTableEntity.tsx b/datahub-web-react/src/app/entity/mlFeatureTable/MLFeatureTableEntity.tsx index 52ea176e07807..3bb54b739e749 100644 --- a/datahub-web-react/src/app/entity/mlFeatureTable/MLFeatureTableEntity.tsx +++ b/datahub-web-react/src/app/entity/mlFeatureTable/MLFeatureTableEntity.tsx @@ -94,16 +94,16 @@ export class MLFeatureTableEntity implements Entity { component: SidebarAboutSection, }, { - component: SidebarTagsSection, + component: SidebarOwnerSection, properties: { - hasTags: true, - hasTerms: true, + defaultOwnerType: OwnershipType.TechnicalOwner, }, }, { - component: SidebarOwnerSection, + component: SidebarTagsSection, properties: { - defaultOwnerType: OwnershipType.TechnicalOwner, + hasTags: true, + hasTerms: true, }, }, { diff --git a/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx b/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx index 94cd9c6d87ad4..3e800f4f733d2 100644 --- a/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx +++ b/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx @@ -99,16 +99,16 @@ export class MLModelEntity implements Entity { component: SidebarAboutSection, }, { - component: SidebarTagsSection, + component: SidebarOwnerSection, properties: { - hasTags: true, - hasTerms: true, + defaultOwnerType: OwnershipType.TechnicalOwner, }, }, { - component: SidebarOwnerSection, + component: SidebarTagsSection, properties: { - defaultOwnerType: OwnershipType.TechnicalOwner, + hasTags: true, + hasTerms: true, }, }, { diff --git a/datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx b/datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx index 4d19a106bc570..1282eab47cefc 100644 --- a/datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx +++ b/datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx @@ -82,16 +82,16 @@ export class MLModelGroupEntity implements Entity { component: SidebarAboutSection, }, { - component: SidebarTagsSection, + component: SidebarOwnerSection, properties: { - hasTags: true, - hasTerms: true, + defaultOwnerType: OwnershipType.TechnicalOwner, }, }, { - component: SidebarOwnerSection, + component: SidebarTagsSection, properties: { - defaultOwnerType: OwnershipType.TechnicalOwner, + hasTags: true, + hasTerms: true, }, }, { diff --git a/datahub-web-react/src/app/entity/mlPrimaryKey/MLPrimaryKeyEntity.tsx b/datahub-web-react/src/app/entity/mlPrimaryKey/MLPrimaryKeyEntity.tsx index 5f0955164fe3e..c6b4bba46f331 100644 --- a/datahub-web-react/src/app/entity/mlPrimaryKey/MLPrimaryKeyEntity.tsx +++ b/datahub-web-react/src/app/entity/mlPrimaryKey/MLPrimaryKeyEntity.tsx @@ -89,16 +89,16 @@ export class MLPrimaryKeyEntity implements Entity { component: SidebarAboutSection, }, { - component: SidebarTagsSection, + component: SidebarOwnerSection, properties: { - hasTags: true, - hasTerms: true, + defaultOwnerType: OwnershipType.TechnicalOwner, }, }, { - component: SidebarOwnerSection, + component: SidebarTagsSection, properties: { - defaultOwnerType: OwnershipType.TechnicalOwner, + hasTags: true, + hasTerms: true, }, }, { From 71f80b6207cc4b6b3820ab95dc20f0a88bbf817b Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Tue, 11 Jul 2023 20:57:29 -0400 Subject: [PATCH 125/222] test(ingest/testing-utils): Add back delta info ignore path (#8402) --- .../datahub/testing/compare_metadata_json.py | 2 +- .../src/datahub/testing/mcp_diff.py | 30 ++++++++++++++----- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/metadata-ingestion/src/datahub/testing/compare_metadata_json.py b/metadata-ingestion/src/datahub/testing/compare_metadata_json.py index 30a6db6871873..5b65ba6006838 100644 --- a/metadata-ingestion/src/datahub/testing/compare_metadata_json.py +++ b/metadata-ingestion/src/datahub/testing/compare_metadata_json.py @@ -89,7 +89,7 @@ def diff_metadata_json( golden: MetadataJson, ignore_paths: Sequence[str] = (), ) -> Union[DeepDiff, MCPDiff]: - ignore_paths = (*ignore_paths, *default_exclude_paths) + ignore_paths = (*ignore_paths, *default_exclude_paths, r"root\[\d+].delta_info") try: golden_map = get_aspects_by_urn(golden) output_map = get_aspects_by_urn(output) diff --git a/metadata-ingestion/src/datahub/testing/mcp_diff.py b/metadata-ingestion/src/datahub/testing/mcp_diff.py index f7aeb6f829012..c253e56a3a707 100644 --- a/metadata-ingestion/src/datahub/testing/mcp_diff.py +++ b/metadata-ingestion/src/datahub/testing/mcp_diff.py @@ -1,6 +1,7 @@ +import dataclasses +import json import re from collections import defaultdict -from dataclasses import dataclass, field from typing import Any, Dict, List, Sequence, Set, Tuple, Union import deepdiff.serialization @@ -27,13 +28,13 @@ ] -@dataclass(frozen=True) +@dataclasses.dataclass(frozen=True) class AspectForDiff: urn: str change_type: str aspect_name: str - aspect: Dict[str, Any] = field(hash=False) - delta_info: "DeltaInfo" = field(hash=False) + aspect: Dict[str, Any] = dataclasses.field(hash=False) + delta_info: "DeltaInfo" = dataclasses.field(hash=False, repr=False) @classmethod def create_from_mcp(cls, idx: int, obj: Dict[str, Any]) -> "AspectForDiff": @@ -46,8 +47,17 @@ def create_from_mcp(cls, idx: int, obj: Dict[str, Any]) -> "AspectForDiff": delta_info=DeltaInfo(idx=idx, original=obj), ) + def __repr__(self): + d = { + field.name: getattr(self, field.name) + for field in dataclasses.fields(self) + if field.repr + } + d["aspect"] = "" + return "\n" + "\t" * 2 + str(json.dumps(d)) -@dataclass + +@dataclasses.dataclass class DeltaInfo: """Information about an MCP used to construct a diff delta. @@ -59,6 +69,10 @@ class DeltaInfo: class DeltaInfoOperator(BaseOperator): + """Warning: Doesn't seem to be working right now. + Ignored via an ignore path as an extra layer of defense. + """ + def __init__(self): super().__init__(types=[DeltaInfo]) @@ -93,7 +107,7 @@ def get_aspects_by_urn(obj: object) -> AspectsByUrn: return d -@dataclass +@dataclasses.dataclass class MCPAspectDiff: aspects_added: Dict[int, AspectForDiff] aspects_removed: Dict[int, AspectForDiff] @@ -126,7 +140,7 @@ def create(cls, diff: DeepDiff) -> "MCPAspectDiff": ) -@dataclass +@dataclasses.dataclass class MCPDiff: aspect_changes: Dict[str, Dict[str, MCPAspectDiff]] # urn -> aspect -> diff urns_added: Set[str] @@ -169,7 +183,7 @@ def create( def convert_path(path: str) -> str: # Attempt to use paths intended for the root golden... sorry for the regex return re.sub( - r"root\\?\[([0-9]+|\\d\+)\\?]\\?\['aspect'\\?](\\?\['(json|value)'\\?])?", + r"^root\\?\[([0-9]+|\\d\+)\\?]\\?\['aspect'\\?](\\?\['(json|value)'\\?])?", r"root\[\\d+].aspect", path, ) From ffd14eebf1182b3b593e9b80f118c3e8a1f8c2ec Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 11 Jul 2023 18:30:57 -0700 Subject: [PATCH 126/222] fix(ingest/bigquery): skip self-references when generating lineage (#8403) --- .../src/datahub/ingestion/source/bigquery_v2/lineage.py | 4 ++++ metadata-ingestion/src/datahub/ingestion/source/tableau.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py index 68cd9051ab5b7..b5c8c74acc089 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py @@ -681,6 +681,10 @@ def get_upstream_tables( for upstream_lineage in lineage_metadata[str(bq_table)]: upstream_table_ref = upstream_lineage.table upstream_table = BigQueryTableRef.from_string_name(upstream_table_ref) + if bq_table == upstream_table: + # Skip self-references. + continue + if upstream_table.is_temporary_table( [self.config.temp_table_dataset_prefix] ): diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau.py index 92f61870aa968..7d1ead5e46b5a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau.py @@ -413,7 +413,7 @@ def update_table( @platform_name("Tableau") @config_class(TableauConfig) -@support_status(SupportStatus.INCUBATING) +@support_status(SupportStatus.CERTIFIED) @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") @capability(SourceCapability.DOMAINS, "Requires transformer", supported=False) @capability(SourceCapability.DESCRIPTIONS, "Enabled by default") From e53d220961de22eb1b370e187f5731325909afcd Mon Sep 17 00:00:00 2001 From: Sheerapthinath Date: Wed, 12 Jul 2023 08:47:57 +0530 Subject: [PATCH 127/222] feat(ingest): datamodel to ingest organisation role metadata for a dataset (#8267) Co-authored-by: John Joyce --- .../datahub/graphql/GmsGraphQLEngine.java | 25 ++++ .../graphql/resolvers/EntityTypeMapper.java | 1 + .../graphql/resolvers/search/SearchUtils.java | 6 +- .../common/mappers/UrnToEntityMapper.java | 6 + .../graphql/types/dataset/DatasetType.java | 3 +- .../types/dataset/mappers/DatasetMapper.java | 4 + .../graphql/types/rolemetadata/RoleType.java | 120 ++++++++++++++++++ .../rolemetadata/mappers/AccessMapper.java | 41 ++++++ .../rolemetadata/mappers/RoleMapper.java | 93 ++++++++++++++ .../src/main/resources/entity.graphql | 98 ++++++++++++++ .../java/com/linkedin/metadata/Constants.java | 8 ++ .../pegasus/com/linkedin/common/Access.pdl | 15 +++ .../com/linkedin/common/RoleAssociation.pdl | 24 ++++ .../com/linkedin/metadata/key/RoleKey.pdl | 14 ++ .../main/pegasus/com/linkedin/role/Actors.pdl | 17 +++ .../com/linkedin/role/RoleProperties.pdl | 42 ++++++ .../pegasus/com/linkedin/role/RoleUser.pdl | 17 +++ .../src/main/resources/entity-registry.yml | 7 + 18 files changed, 538 insertions(+), 3 deletions(-) create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/rolemetadata/RoleType.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/rolemetadata/mappers/AccessMapper.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/rolemetadata/mappers/RoleMapper.java create mode 100644 metadata-models/src/main/pegasus/com/linkedin/common/Access.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/common/RoleAssociation.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/key/RoleKey.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/role/Actors.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/role/RoleProperties.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/role/RoleUser.pdl diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 378362a77654c..1dd07c1873b2f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -280,6 +280,7 @@ import com.linkedin.datahub.graphql.types.dataset.VersionedDatasetType; import com.linkedin.datahub.graphql.types.dataset.mappers.DatasetProfileMapper; import com.linkedin.datahub.graphql.types.domain.DomainType; +import com.linkedin.datahub.graphql.types.rolemetadata.RoleType; import com.linkedin.datahub.graphql.types.glossary.GlossaryNodeType; import com.linkedin.datahub.graphql.types.glossary.GlossaryTermType; import com.linkedin.datahub.graphql.types.mlmodel.MLFeatureTableType; @@ -395,6 +396,9 @@ public class GmsGraphQLEngine { private final ViewsConfiguration viewsConfiguration; private final DatasetType datasetType; + + private final RoleType roleType; + private final CorpUserType corpUserType; private final CorpGroupType corpGroupType; private final ChartType chartType; @@ -503,6 +507,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { this.featureFlags = args.featureFlags; this.datasetType = new DatasetType(entityClient); + this.roleType = new RoleType(entityClient); this.corpUserType = new CorpUserType(entityClient, featureFlags); this.corpGroupType = new CorpGroupType(entityClient); this.chartType = new ChartType(entityClient); @@ -538,6 +543,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { // Init Lists this.entityTypes = ImmutableList.of( datasetType, + roleType, corpUserType, corpGroupType, dataPlatformType, @@ -634,6 +640,7 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) { configureContainerResolvers(builder); configureDataPlatformInstanceResolvers(builder); configureGlossaryTermResolvers(builder); + configureOrganisationRoleResolvers(builder); configureGlossaryNodeResolvers(builder); configureDomainResolvers(builder); configureAssertionResolvers(builder); @@ -651,6 +658,23 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) { configurePluginResolvers(builder); } + private void configureOrganisationRoleResolvers(RuntimeWiring.Builder builder) { + builder.type("Role", typeWiring -> typeWiring + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + ); + builder.type("RoleAssociation", typeWiring -> typeWiring + .dataFetcher("role", + new LoadableTypeResolver<>(roleType, + (env) -> ((com.linkedin.datahub.graphql.generated.RoleAssociation) + env.getSource()).getRole().getUrn())) + ); + builder.type("RoleUser", typeWiring -> typeWiring + .dataFetcher("user", + new LoadableTypeResolver<>(corpUserType, + (env) -> ((com.linkedin.datahub.graphql.generated.RoleUser) + env.getSource()).getUser().getUrn())) + ); + } public GraphQLEngine.Builder builder() { final GraphQLEngine.Builder builder = GraphQLEngine.builder(); @@ -771,6 +795,7 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("browse", new BrowseResolver(browsableTypes)) .dataFetcher("browsePaths", new BrowsePathsResolver(browsableTypes)) .dataFetcher("dataset", getResolver(datasetType)) + .dataFetcher("role", getResolver(roleType)) .dataFetcher("versionedDataset", getResolver(versionedDatasetType, (env) -> new VersionedUrn().setUrn(UrnUtils.getUrn(env.getArgument(URN_FIELD_NAME))) .setVersionStamp(env.getArgument(VERSION_STAMP_FIELD_NAME)))) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java index 9a72d7dc2c77d..3682b2282544e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java @@ -16,6 +16,7 @@ public class EntityTypeMapper { static final Map ENTITY_TYPE_TO_NAME = ImmutableMap.builder() .put(EntityType.DATASET, "dataset") + .put(EntityType.ROLE, "role") .put(EntityType.CORP_USER, "corpuser") .put(EntityType.CORP_GROUP, "corpGroup") .put(EntityType.DATA_PLATFORM, "dataPlatform") diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java index 492380aa21482..3b516ce8ca8e6 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java @@ -70,8 +70,10 @@ private SearchUtils() { EntityType.CORP_GROUP, EntityType.CONTAINER, EntityType.DOMAIN, - EntityType.NOTEBOOK, - EntityType.DATA_PRODUCT); + EntityType.DATA_PRODUCT, + EntityType.ROLE, + EntityType.NOTEBOOK); + /** * Entities that are part of autocomplete by default in Auto Complete Across Entities diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java index 2f3204aae0afe..34bf56a396b62 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java @@ -28,6 +28,7 @@ import com.linkedin.datahub.graphql.generated.MLPrimaryKey; import com.linkedin.datahub.graphql.generated.Notebook; import com.linkedin.datahub.graphql.generated.OwnershipTypeEntity; +import com.linkedin.datahub.graphql.generated.Role; import com.linkedin.datahub.graphql.generated.SchemaFieldEntity; import com.linkedin.datahub.graphql.generated.Tag; import com.linkedin.datahub.graphql.generated.Test; @@ -52,6 +53,11 @@ public Entity apply(Urn input) { ((Dataset) partialEntity).setUrn(input.toString()); ((Dataset) partialEntity).setType(EntityType.DATASET); } + if (input.getEntityType().equals("role")) { + partialEntity = new Role(); + ((Role) partialEntity).setUrn(input.toString()); + ((Role) partialEntity).setType(EntityType.ROLE); + } if (input.getEntityType().equals("glossaryTerm")) { partialEntity = new GlossaryTerm(); ((GlossaryTerm) partialEntity).setUrn(input.toString()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java index 510971c8db11c..0fc4399ac902d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java @@ -85,7 +85,8 @@ public class DatasetType implements SearchableEntityType, Brows SIBLINGS_ASPECT_NAME, EMBED_ASPECT_NAME, DATA_PRODUCTS_ASPECT_NAME, - BROWSE_PATHS_V2_ASPECT_NAME + BROWSE_PATHS_V2_ASPECT_NAME, + ACCESS_DATASET_ASPECT_NAME ); private static final Set FACET_FIELDS = ImmutableSet.of("origin", "platform"); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java index 16c2750cf87e0..f0899f8fbc0cb 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql.types.dataset.mappers; +import com.linkedin.common.Access; import com.linkedin.common.BrowsePathsV2; import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.Deprecation; @@ -32,6 +33,7 @@ import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils; import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper; +import com.linkedin.datahub.graphql.types.rolemetadata.mappers.AccessMapper; import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; @@ -110,6 +112,8 @@ public Dataset apply(@Nonnull final EntityResponse entityResponse) { dataset.setEmbed(EmbedMapper.map(new Embed(dataMap)))); mappingHelper.mapToResult(BROWSE_PATHS_V2_ASPECT_NAME, (dataset, dataMap) -> dataset.setBrowsePathV2(BrowsePathsV2Mapper.map(new BrowsePathsV2(dataMap)))); + mappingHelper.mapToResult(ACCESS_DATASET_ASPECT_NAME, ((dataset, dataMap) -> + dataset.setAccess(AccessMapper.map(new Access(dataMap), entityUrn)))); return mappingHelper.getResult(); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/rolemetadata/RoleType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/rolemetadata/RoleType.java new file mode 100644 index 0000000000000..084c4d5033ad0 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/rolemetadata/RoleType.java @@ -0,0 +1,120 @@ +package com.linkedin.datahub.graphql.types.rolemetadata; + +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.AutoCompleteResults; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.FacetFilterInput; +import com.linkedin.datahub.graphql.generated.Role; +import com.linkedin.datahub.graphql.generated.SearchResults; +import com.linkedin.datahub.graphql.types.SearchableEntityType; +import com.linkedin.datahub.graphql.types.rolemetadata.mappers.RoleMapper; +import com.linkedin.datahub.graphql.types.mappers.AutoCompleteResultsMapper; +import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.query.AutoCompleteResult; +import com.linkedin.metadata.query.SearchFlags; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.search.SearchResult; +import graphql.execution.DataFetcherResult; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class RoleType implements SearchableEntityType, + com.linkedin.datahub.graphql.types.EntityType { + + static final Set ASPECTS_TO_FETCH = ImmutableSet.of( + Constants.ROLE_KEY, + Constants.ROLE_PROPERTIES_ASPECT_NAME, + Constants.ROLE_ACTORS_ASPECT_NAME + ); + + private final EntityClient _entityClient; + + public RoleType(final EntityClient entityClient) { + _entityClient = entityClient; + } + + @Override + public EntityType type() { + return EntityType.ROLE; + } + + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + + @Override + public Class objectClass() { + return Role.class; + } + + @Override + public List> batchLoad(@Nonnull List urns, + @Nonnull QueryContext context) throws Exception { + final List externalRolesUrns = urns.stream() + .map(UrnUtils::getUrn) + .collect(Collectors.toList()); + + try { + final Map entities = _entityClient.batchGetV2( + Constants.ROLE_ENTITY_NAME, + new HashSet<>(externalRolesUrns), + ASPECTS_TO_FETCH, + context.getAuthentication()); + + final List gmsResults = new ArrayList<>(); + for (Urn urn : externalRolesUrns) { + gmsResults.add(entities.getOrDefault(urn, null)); + } + return gmsResults.stream() + .map(gmsResult -> + gmsResult == null ? null : DataFetcherResult.newResult() + .data(RoleMapper.map(gmsResult)) + .build() + ) + .collect(Collectors.toList()); + } catch (Exception e) { + throw new RuntimeException("Failed to batch load Role", e); + } + } + + @Override + public SearchResults search(@Nonnull String query, + @Nullable List filters, + int start, + int count, + @Nonnull final QueryContext context) throws Exception { + final SearchResult searchResult = _entityClient.search(Constants.ROLE_ENTITY_NAME, + query, Collections.emptyMap(), start, count, + context.getAuthentication(), new SearchFlags().setFulltext(true)); + return UrnSearchResultsMapper.map(searchResult); + } + + @Override + public AutoCompleteResults autoComplete(@Nonnull String query, + @Nullable String field, + @Nullable Filter filters, + int limit, + @Nonnull final QueryContext context) throws Exception { + final AutoCompleteResult result = _entityClient.autoComplete(Constants.ROLE_ENTITY_NAME, + query, filters, limit, context.getAuthentication()); + return AutoCompleteResultsMapper.map(result); + } + +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/rolemetadata/mappers/AccessMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/rolemetadata/mappers/AccessMapper.java new file mode 100644 index 0000000000000..cabace1a52441 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/rolemetadata/mappers/AccessMapper.java @@ -0,0 +1,41 @@ +package com.linkedin.datahub.graphql.types.rolemetadata.mappers; + + +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.Role; +import com.linkedin.datahub.graphql.generated.RoleAssociation; + +import javax.annotation.Nonnull; +import java.util.stream.Collectors; + +public class AccessMapper { + public static final AccessMapper INSTANCE = new AccessMapper(); + + public static com.linkedin.datahub.graphql.generated.Access map( + @Nonnull final com.linkedin.common.Access access, + @Nonnull final Urn entityUrn) { + return INSTANCE.apply(access, entityUrn); + } + + public com.linkedin.datahub.graphql.generated.Access apply( + @Nonnull final com.linkedin.common.Access access, + @Nonnull final Urn entityUrn) { + com.linkedin.datahub.graphql.generated.Access result = new com.linkedin.datahub.graphql.generated.Access(); + result.setRoles(access.getRoles().stream().map( + association -> this.mapRoleAssociation(association, entityUrn) + ).collect(Collectors.toList())); + return result; + } + + private RoleAssociation mapRoleAssociation(com.linkedin.common.RoleAssociation association, Urn entityUrn) { + RoleAssociation roleAssociation = new RoleAssociation(); + Role role = new Role(); + role.setType(EntityType.ROLE); + role.setUrn(association.getUrn().toString()); + roleAssociation.setRole(role); + roleAssociation.setAssociatedUrn(entityUrn.toString()); + return roleAssociation; + } + +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/rolemetadata/mappers/RoleMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/rolemetadata/mappers/RoleMapper.java new file mode 100644 index 0000000000000..3cb0ec942a457 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/rolemetadata/mappers/RoleMapper.java @@ -0,0 +1,93 @@ +package com.linkedin.datahub.graphql.types.rolemetadata.mappers; + +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.generated.Actor; +import com.linkedin.datahub.graphql.generated.CorpUser; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.Role; +import com.linkedin.datahub.graphql.generated.RoleProperties; +import com.linkedin.datahub.graphql.generated.RoleUser; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.key.RoleKey; +import com.linkedin.role.Actors; +import com.linkedin.role.RoleUserArray; + +import javax.annotation.Nonnull; +import java.util.List; +import java.util.stream.Collectors; + +public class RoleMapper implements ModelMapper { + + public static final RoleMapper INSTANCE = new RoleMapper(); + + public static Role map(@Nonnull final EntityResponse entityResponse) { + return INSTANCE.apply(entityResponse); + } + + private static RoleProperties mapRoleProperties(final com.linkedin.role.RoleProperties e) { + final RoleProperties propertiesResult = new RoleProperties(); + propertiesResult.setName(e.getName()); + propertiesResult.setDescription(e.getDescription()); + propertiesResult.setType(e.getType()); + propertiesResult.setRequestUrl(e.getRequestUrl()); + + return propertiesResult; + } + + private static RoleUser mapCorpUsers(final com.linkedin.role.RoleUser provisionedUser) { + RoleUser result = new RoleUser(); + CorpUser corpUser = new CorpUser(); + corpUser.setUrn(provisionedUser.getUser().toString()); + result.setUser(corpUser); + return result; + } + + private static Actor mapActor(Actors actors) { + Actor actor = new Actor(); + actor.setUsers(mapRoleUsers(actors.getUsers())); + return actor; + } + + private static List mapRoleUsers(RoleUserArray users) { + if (users == null) { + return null; + } + return users.stream().map(x -> mapCorpUsers(x)).collect(Collectors.toList()); + } + + @Override + public Role apply(EntityResponse input) { + + + final Role result = new Role(); + final Urn entityUrn = input.getUrn(); + + result.setUrn(entityUrn.toString()); + result.setType(EntityType.ROLE); + + final EnvelopedAspectMap aspects = input.getAspects(); + + final EnvelopedAspect roleKeyAspect = aspects.get(Constants.ROLE_KEY); + if (roleKeyAspect != null) { + result.setId(new RoleKey(roleKeyAspect.getValue().data()).getId()); + } + final EnvelopedAspect envelopedPropertiesAspect = aspects.get(Constants.ROLE_PROPERTIES_ASPECT_NAME); + if (envelopedPropertiesAspect != null) { + result.setProperties(mapRoleProperties( + new com.linkedin.role.RoleProperties( + envelopedPropertiesAspect.getValue().data())) + ); + } + + final EnvelopedAspect envelopedUsers = aspects.get(Constants.ROLE_ACTORS_ASPECT_NAME); + if (envelopedUsers != null) { + result.setActors(mapActor(new Actors(envelopedUsers.getValue().data()))); + } + + return result; + } +} diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index e45ecfe519d8b..59e35d5bc3ff4 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -886,6 +886,11 @@ enum EntityType { A Custom Ownership Type """ CUSTOM_OWNERSHIP_TYPE + + """" + A Role from an organisation + """ + ROLE } """ @@ -1260,6 +1265,11 @@ type Dataset implements EntityWithRelationships & Entity & BrowsableEntity { """ domain: DomainAssociation + """ + The Roles and the properties to access the dataset + """ + access: Access + """ Statistics about how this Dataset is used The first parameter, `resource`, is deprecated and no longer needs to be provided @@ -1399,6 +1409,94 @@ type Dataset implements EntityWithRelationships & Entity & BrowsableEntity { exists: Boolean } +type RoleAssociation { + + """ + The Role entity itself + """ + role: Role! + + """ + Reference back to the tagged urn for tracking purposes e.g. when sibling nodes are merged together + """ + associatedUrn: String! + +} + +type Access { + roles: [RoleAssociation!] +} + +type Role implements Entity { + """ + A primary key of the Metadata Entity + """ + urn: String! + + """ + A standard Entity Type + """ + type: EntityType! + + """ + List of relationships between the source Entity and some destination entities with a given types + """ + relationships(input: RelationshipsInput!): EntityRelationshipsResult + + """ + Id of the Role + """ + id: String! + + """ + Role properties to include Request Access Url + """ + properties: RoleProperties! + + """ + A standard Entity Type + """ + actors: Actor! + + +} + +type Actor { + """ + List of users for which the role is provisioned + """ + users: [RoleUser!] +} + +type RoleUser { + """ + Linked corp user of a role + """ + user: CorpUser! +} + +type RoleProperties { + """ + Name of the Role in an organisation + """ + name: String! + + """ + Description about the role + """ + description: String + + """ + Role type can be READ, WRITE or ADMIN + """ + type: String + + """ + Url to request a role for a user in an organisation + """ + requestUrl: String +} + type FineGrainedLineage { upstreams: [SchemaFieldRef!] downstreams: [SchemaFieldRef!] diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java index 184c999c81b04..d37ced007133c 100644 --- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -228,6 +228,14 @@ public class Constants { public static final String DOMAIN_KEY_ASPECT_NAME = "domainKey"; public static final String DOMAIN_PROPERTIES_ASPECT_NAME = "domainProperties"; public static final String DOMAINS_ASPECT_NAME = "domains"; + + // ExternalRoleMetadata + public static final String ROLE_ENTITY_NAME = "role"; + public static final String ACCESS_DATASET_ASPECT_NAME = "access"; + public static final String ROLE_KEY = "roleKey"; + public static final String ROLE_PROPERTIES_ASPECT_NAME = "roleProperties"; + public static final String ROLE_ACTORS_ASPECT_NAME = "actors"; + public static final String DOMAIN_CREATED_TIME_INDEX_FIELD_NAME = "createdTime"; // Assertion diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/Access.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/Access.pdl new file mode 100644 index 0000000000000..bff85b77823c7 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/common/Access.pdl @@ -0,0 +1,15 @@ +namespace com.linkedin.common + + +/** + * Aspect used for associating roles to a dataset or any asset + */ +@Aspect = { + "name": "access" +} +record Access { + /** + * List of Roles which needs to be associated + */ + roles: optional array[RoleAssociation] +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/RoleAssociation.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/RoleAssociation.pdl new file mode 100644 index 0000000000000..ddd63ed64014b --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/common/RoleAssociation.pdl @@ -0,0 +1,24 @@ +namespace com.linkedin.common + + +/** + * Properties of an applied Role. For now, just an Urn + */ +record RoleAssociation { + + /** + * Urn of the External Role + */ + @Relationship = { + "name": "AssociatedWith", + "entityTypes": [ "role" ] + } + @Searchable = { + "fieldName": "roles", + "fieldType": "URN", + "hasValuesFieldName": "hasRoles", + "addToFilters": true, + "filterNameOverride": "Role" + } + urn: Urn +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/key/RoleKey.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/RoleKey.pdl new file mode 100644 index 0000000000000..8d94d83b9a83f --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/RoleKey.pdl @@ -0,0 +1,14 @@ +namespace com.linkedin.metadata.key + +/** + * Key for a External AccessManagement + */ +@Aspect = { + "name": "roleKey" +} +record RoleKey { + /** + * A unique id for the access management IAM. + */ + id: string +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/role/Actors.pdl b/metadata-models/src/main/pegasus/com/linkedin/role/Actors.pdl new file mode 100644 index 0000000000000..80e1ab0702cb9 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/role/Actors.pdl @@ -0,0 +1,17 @@ +namespace com.linkedin.role + + +/** + * Provisioned users of a role + */ +@Aspect = { + "name": "actors" +} +record Actors { + + /** + * List of provisioned users of a role + */ + users: optional array[RoleUser] + +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/role/RoleProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/role/RoleProperties.pdl new file mode 100644 index 0000000000000..acebdf5558c59 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/role/RoleProperties.pdl @@ -0,0 +1,42 @@ +namespace com.linkedin.role + +import com.linkedin.common.AuditStamp + +/** + * Information about a ExternalRoleProperties + */ +@Aspect = { + "name": "roleProperties" +} +record RoleProperties { + + /** + * Display name of the IAM Role in the external system + */ + @Searchable = { + "fieldType": "TEXT_PARTIAL", + "enableAutocomplete": true, + "boostScore": 10.0 + } + name: string + + /** + * Description of the IAM Role + */ + description: optional string + + /** + * Can be READ, ADMIN, WRITE + */ + type: string + + /** + * Link to access external access management + */ + requestUrl: optional string + + /** + * Created Audit stamp + */ + created: optional AuditStamp +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/role/RoleUser.pdl b/metadata-models/src/main/pegasus/com/linkedin/role/RoleUser.pdl new file mode 100644 index 0000000000000..ba66f42694574 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/role/RoleUser.pdl @@ -0,0 +1,17 @@ +namespace com.linkedin.role + +import com.linkedin.common.Urn +/** + * Provisioned users of a role + */ +record RoleUser { + + /** + * Link provisioned corp user for a role + */ + @Relationship = { + "name": "Has", + "entityTypes": [ "corpuser" ] + } + user: Urn +} diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml index 9eae7593851cb..5f54525e1e862 100644 --- a/metadata-models/src/main/resources/entity-registry.yml +++ b/metadata-models/src/main/resources/entity-registry.yml @@ -4,6 +4,12 @@ entities: keyAspect: dataPlatformKey aspects: - dataPlatformInfo + - name: role + category: core + keyAspect: roleKey + aspects: + - roleProperties + - actors - name: dataset doc: Datasets represent logical or physical data assets stored or represented in various data platforms. Tables, Views, Streams are all instances of datasets. category: core @@ -35,6 +41,7 @@ entities: - browsePaths - dataPlatformInstance - browsePathsV2 + - access - name: dataHubPolicy doc: DataHub Policies represent access policies granted to users or groups on metadata operations like edit, view etc. category: internal From ea3e119210802cadc886dd28bafa7eda093af39f Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Wed, 12 Jul 2023 00:46:10 -0400 Subject: [PATCH 128/222] test(ingest/kafka-connect): Attempt to fix flaky test (#8404) --- .../kafka-connect/test_kafka_connect.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/metadata-ingestion/tests/integration/kafka-connect/test_kafka_connect.py b/metadata-ingestion/tests/integration/kafka-connect/test_kafka_connect.py index aaa39cd5f52d9..49dada0cddcbb 100644 --- a/metadata-ingestion/tests/integration/kafka-connect/test_kafka_connect.py +++ b/metadata-ingestion/tests/integration/kafka-connect/test_kafka_connect.py @@ -34,6 +34,13 @@ def is_mysql_up(container_name: str, port: int) -> bool: return ret.returncode == 0 +def have_connectors_processed(container_name: str) -> bool: + """A cheap way to figure out if postgres is responsive on a container""" + + cmd = f"docker logs {container_name} 2>&1 | grep 'Session key updated'" + return subprocess.run(cmd, shell=True).returncode == 0 + + @pytest.fixture(scope="module") def kafka_connect_runner(docker_compose_runner, pytestconfig, test_resources_dir): test_resources_dir_kafka = pytestconfig.rootpath / "tests/integration/kafka" @@ -66,10 +73,7 @@ def kafka_connect_runner(docker_compose_runner, pytestconfig, test_resources_dir docker_services.wait_until_responsive( timeout=30, pause=1, - check=lambda: requests.get( - KAFKA_CONNECT_ENDPOINT, - ).status_code - == 200, + check=lambda: requests.get(KAFKA_CONNECT_ENDPOINT).status_code == 200, ) yield docker_services @@ -298,6 +302,13 @@ def loaded_kafka_connect(kafka_connect_runner): ) assert ret.returncode == 0 + # Give time for connectors to process the table data + kafka_connect_runner.wait_until_responsive( + timeout=30, + pause=1, + check=lambda: have_connectors_processed("test_connect"), + ) + @freeze_time(FROZEN_TIME) @pytest.mark.integration_batch_1 From ae7e3c208082a9c7674e727ca50a3c9a2968b70d Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 12 Jul 2023 04:41:13 -0700 Subject: [PATCH 129/222] feat(ingest/dbt-cloud): reduce graphql query complexity (#8390) --- .../datahub/ingestion/source/dbt/dbt_cloud.py | 92 ++++++++++--------- 1 file changed, 49 insertions(+), 43 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py index 462444ec0d5a1..1cd5ed8164854 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py @@ -104,23 +104,20 @@ class DBTCloudConfig(DBTCommonConfig): compiledCode """ -_DBT_GRAPHQL_QUERY = f""" -query DatahubMetadataQuery($jobId: Int!, $runId: Int) {{ - models(jobId: $jobId, runId: $runId) {{ +_DBT_FIELDS_BY_TYPE = { + "models": f""" { _DBT_GRAPHQL_COMMON_FIELDS } { _DBT_GRAPHQL_NODE_COMMON_FIELDS } { _DBT_GRAPHQL_MODEL_SEED_SNAPSHOT_FIELDS } dependsOn materializedType - }} - - seeds(jobId: $jobId, runId: $runId) {{ +""", + "seeds": f""" { _DBT_GRAPHQL_COMMON_FIELDS } { _DBT_GRAPHQL_NODE_COMMON_FIELDS } { _DBT_GRAPHQL_MODEL_SEED_SNAPSHOT_FIELDS } - }} - - sources(jobId: $jobId, runId: $runId) {{ +""", + "sources": f""" { _DBT_GRAPHQL_COMMON_FIELDS } { _DBT_GRAPHQL_NODE_COMMON_FIELDS } identifier @@ -131,9 +128,8 @@ class DBTCloudConfig(DBTCommonConfig): state freshnessChecked loader - }} - - snapshots(jobId: $jobId, runId: $runId) {{ +""", + "snapshots": f""" { _DBT_GRAPHQL_COMMON_FIELDS } { _DBT_GRAPHQL_NODE_COMMON_FIELDS } { _DBT_GRAPHQL_MODEL_SEED_SNAPSHOT_FIELDS } @@ -143,9 +139,8 @@ class DBTCloudConfig(DBTCommonConfig): parentsModels {{ uniqueId }} - }} - - tests(jobId: $jobId, runId: $runId) {{ +""", + "tests": f""" { _DBT_GRAPHQL_COMMON_FIELDS } state columnName @@ -159,12 +154,18 @@ class DBTCloudConfig(DBTCommonConfig): rawCode compiledSql compiledCode +""", + # Currently unsupported dbt node types: + # - metrics + # - snapshots + # - exposures +} + +_DBT_GRAPHQL_QUERY = """ +query DatahubMetadataQuery_{type}($jobId: Int!, $runId: Int) {{ + {type}(jobId: $jobId, runId: $runId) {{ +{fields} }} - - # Currently unsupported dbt node types: - # - metrics - # - snapshots - # - exposures }} """ @@ -206,15 +207,36 @@ def load_nodes(self) -> Tuple[List[DBTNode], Dict[str, Optional[str]]]: # Additionally, we'd like to model dbt Cloud jobs/runs in DataHub # as DataProcesses or DataJobs. - logger.debug("Sending graphql request to the dbt Cloud metadata API") - response = requests.post( - self.config.metadata_endpoint, - json={ - "query": _DBT_GRAPHQL_QUERY, - "variables": { + raw_nodes = [] + for node_type, fields in _DBT_FIELDS_BY_TYPE.items(): + logger.info(f"Fetching {node_type} from dbt Cloud") + data = self._send_graphql_query( + query=_DBT_GRAPHQL_QUERY.format(type=node_type, fields=fields), + variables={ "jobId": self.config.job_id, "runId": self.config.run_id, }, + ) + + raw_nodes.extend(data[node_type]) + + nodes = [self._parse_into_dbt_node(node) for node in raw_nodes] + + additional_metadata: Dict[str, Optional[str]] = { + "project_id": str(self.config.project_id), + "account_id": str(self.config.account_id), + "job_id": str(self.config.job_id), + } + + return nodes, additional_metadata + + def _send_graphql_query(self, query: str, variables: Dict) -> Dict: + logger.debug(f"Sending GraphQL query to dbt Cloud: {query}") + response = requests.post( + self.config.metadata_endpoint, + json={ + "query": query, + "variables": variables, }, headers={ "Authorization": f"Bearer {self.config.token}", @@ -233,23 +255,7 @@ def load_nodes(self) -> Tuple[List[DBTNode], Dict[str, Optional[str]]]: response.raise_for_status() raise e - raw_nodes = [ - *data["models"], - *data["seeds"], - *data["sources"], - *data["snapshots"], - *data["tests"], - ] - - nodes = [self._parse_into_dbt_node(node) for node in raw_nodes] - - additional_metadata: Dict[str, Optional[str]] = { - "project_id": str(self.config.project_id), - "account_id": str(self.config.account_id), - "job_id": str(self.config.job_id), - } - - return nodes, additional_metadata + return data def _parse_into_dbt_node(self, node: Dict) -> DBTNode: key = node["uniqueId"] From a2fc02294c71ef7a656a7565eb658c383961af8e Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Wed, 12 Jul 2023 17:11:33 +0530 Subject: [PATCH 130/222] fix(ingest/snowflake): fix azure cloud region ids in external url (#8376) Co-authored-by: Harshal Sheth --- .../ingestion/source/snowflake/constants.py | 17 +++++++++++++++-- .../tests/unit/test_snowflake_source.py | 4 ++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/constants.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/constants.py index 05f526a0603ce..6f5e22e39d0c8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/constants.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/constants.py @@ -17,12 +17,25 @@ class SnowflakeEdition(str, Enum): ENTERPRISE = "Enterprise or above" -# See https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#region-ids +# See https://docs.snowflake.com/en/user-guide/admin-account-identifier#non-vps-account-locator-formats-by-cloud-platform-and-region # Includes only exceptions to format _ SNOWFLAKE_REGION_CLOUD_REGION_MAPPING = { "aws_us_east_1_gov": (SnowflakeCloudProvider.AWS, "us-east-1"), + "azure_westus2": (SnowflakeCloudProvider.AZURE, "west-us-2"), + "azure_centralus": (SnowflakeCloudProvider.AZURE, "central-us"), + "azure_southcentralus": (SnowflakeCloudProvider.AZURE, "south-central-us"), + "azure_eastus2": (SnowflakeCloudProvider.AZURE, "east-us-2"), + "azure_usgovvirginia": (SnowflakeCloudProvider.AZURE, "us-gov-virginia"), + "azure_canadacentral": (SnowflakeCloudProvider.AZURE, "canada-central"), "azure_uksouth": (SnowflakeCloudProvider.AZURE, "uk-south"), - "azure_centralindia": (SnowflakeCloudProvider.AZURE, "central-india.azure"), + "azure_northeurope": (SnowflakeCloudProvider.AZURE, "north-europe"), + "azure_westeurope": (SnowflakeCloudProvider.AZURE, "west-europe"), + "azure_switzerlandnorth": (SnowflakeCloudProvider.AZURE, "switzerland-north"), + "azure_uaenorth": (SnowflakeCloudProvider.AZURE, "uae-north"), + "azure_centralindia": (SnowflakeCloudProvider.AZURE, "central-india"), + "azure_japaneast": (SnowflakeCloudProvider.AZURE, "japan-east"), + "azure_southeastasia": (SnowflakeCloudProvider.AZURE, "southeast-asia"), + "azure_australiaeast": (SnowflakeCloudProvider.AZURE, "australia-east"), } # https://docs.snowflake.com/en/sql-reference/snowflake-db.html diff --git a/metadata-ingestion/tests/unit/test_snowflake_source.py b/metadata-ingestion/tests/unit/test_snowflake_source.py index 66f25b83cfe69..1c26ca2487e5c 100644 --- a/metadata-ingestion/tests/unit/test_snowflake_source.py +++ b/metadata-ingestion/tests/unit/test_snowflake_source.py @@ -593,7 +593,7 @@ def test_azure_cloud_region_from_snowflake_region_id(): ) assert cloud == SnowflakeCloudProvider.AZURE - assert cloud_region_id == "switzerlandnorth" + assert cloud_region_id == "switzerland-north" ( cloud, @@ -603,7 +603,7 @@ def test_azure_cloud_region_from_snowflake_region_id(): ) assert cloud == SnowflakeCloudProvider.AZURE - assert cloud_region_id == "central-india.azure" + assert cloud_region_id == "central-india" def test_unknown_cloud_region_from_snowflake_region_id(): From a884cf3f904c4ff98074ed9c690f2e9f23fc8448 Mon Sep 17 00:00:00 2001 From: Indy Prentice Date: Wed, 12 Jul 2023 08:42:23 -0300 Subject: [PATCH 131/222] feat(elasticsearch): optimization to reindex instead of deleteByQuery (#8352) --- .../indexbuilder/ESIndexBuilder.java | 72 +++++++++++++++---- .../timeseries/TimeseriesAspectService.java | 11 +++ .../ElasticSearchTimeseriesAspectService.java | 18 +++++ .../TimeseriesAspectIndexBuilders.java | 29 ++++++++ ...nkedin.operations.operations.restspec.json | 8 +++ ...nkedin.operations.operations.snapshot.json | 8 +++ .../operations/OperationsResource.java | 44 +++++++++--- .../operations/OperationsResourceTest.java | 67 ++++++++++++++++- .../mock/MockTimeseriesAspectService.java | 6 ++ 9 files changed, 237 insertions(+), 26 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java index 0a8a4ef7937ea..9affa4c05a566 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import com.linkedin.metadata.search.utils.ESUtils; +import com.linkedin.metadata.timeseries.BatchWriteOperationsOptions; import com.linkedin.metadata.version.GitVersion; import java.io.IOException; import java.time.Duration; @@ -25,6 +26,7 @@ import io.github.resilience4j.retry.Retry; import io.github.resilience4j.retry.RetryConfig; import io.github.resilience4j.retry.RetryRegistry; +import javax.annotation.Nullable; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.http.client.config.RequestConfig; @@ -48,6 +50,8 @@ import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.tasks.TaskSubmissionResponse; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.reindex.ReindexRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; @@ -220,6 +224,28 @@ public void buildIndex(ReindexConfig indexState) throws IOException { } } + public String reindexInPlaceAsync(String indexAlias, @Nullable QueryBuilder filterQuery, BatchWriteOperationsOptions options, ReindexConfig config) + throws Exception { + GetAliasesResponse aliasesResponse = _searchClient.indices().getAlias( + new GetAliasesRequest(indexAlias), RequestOptions.DEFAULT); + if (aliasesResponse.getAliases().isEmpty()) { + throw new IllegalArgumentException(String.format("Input to reindexInPlaceAsync should be an alias. %s is not", indexAlias)); + } + + // Point alias at new index + String nextIndexName = getNextIndexName(indexAlias, System.currentTimeMillis()); + createIndex(nextIndexName, config); + renameReindexedIndices(_searchClient, indexAlias, null, nextIndexName, false); + + return submitReindex(aliasesResponse.getAliases().keySet().toArray(new String[0]), + nextIndexName, options.getBatchSize(), + TimeValue.timeValueSeconds(options.getTimeoutSeconds()), filterQuery); + } + + private static String getNextIndexName(String base, long startTime) { + return base + "_" + startTime; + } + private void reindex(ReindexConfig indexState) throws Throwable { final long startTime = System.currentTimeMillis(); @@ -228,7 +254,7 @@ private void reindex(ReindexConfig indexState) throws Throwable { final long finalCheckIntervalMilli = 60000; final long timeoutAt = startTime + (1000 * 60 * 60 * maxReindexHours); - String tempIndexName = indexState.name() + "_" + startTime; + String tempIndexName = getNextIndexName(indexState.name(), startTime); try { Optional previousTaskInfo = getTaskInfoByHeader(indexState.name()); @@ -309,14 +335,18 @@ private void reindex(ReindexConfig indexState) throws Throwable { } log.info("Reindex from {} to {} succeeded", indexState.name(), tempIndexName); - renameReindexedIndices(_searchClient, indexState.name(), indexState.indexPattern(), tempIndexName); + renameReindexedIndices(_searchClient, indexState.name(), indexState.indexPattern(), tempIndexName, true); log.info("Finished setting up {}", indexState.name()); } - public static void renameReindexedIndices(RestHighLevelClient searchClient, String originalName, String pattern, String newName) + public static void renameReindexedIndices(RestHighLevelClient searchClient, String originalName, @Nullable String pattern, String newName, boolean deleteOld) throws IOException { + GetAliasesRequest getAliasesRequest = new GetAliasesRequest(originalName); + if (pattern != null) { + getAliasesRequest.indices(pattern); + } GetAliasesResponse aliasesResponse = searchClient.indices().getAlias( - new GetAliasesRequest(originalName).indices(pattern), RequestOptions.DEFAULT); + getAliasesRequest, RequestOptions.DEFAULT); // If not aliased, delete the original index final Collection aliasedIndexDelete; @@ -329,28 +359,40 @@ public static void renameReindexedIndices(RestHighLevelClient searchClient, Stri } // Add alias for the new index - AliasActions removeAction = AliasActions.removeIndex() - .indices(aliasedIndexDelete.toArray(new String[0])); + AliasActions removeAction = deleteOld ? AliasActions.removeIndex() : AliasActions.remove().alias(originalName); + removeAction.indices(aliasedIndexDelete.toArray(new String[0])); AliasActions addAction = AliasActions.add().alias(originalName).index(newName); searchClient.indices() .updateAliases(new IndicesAliasesRequest().addAliasAction(removeAction).addAliasAction(addAction), RequestOptions.DEFAULT); } - private String submitReindex(String sourceIndex, String destinationIndex) throws IOException { + private String submitReindex(String[] sourceIndices, String destinationIndex, + int batchSize, @Nullable TimeValue timeout, + @Nullable QueryBuilder sourceFilterQuery) throws IOException { ReindexRequest reindexRequest = new ReindexRequest() - .setSourceIndices(sourceIndex) - .setDestIndex(destinationIndex) - .setMaxRetries(numRetries) - .setAbortOnVersionConflict(false) - .setSourceBatchSize(2500); - - RequestOptions requestOptions = ESUtils.buildReindexTaskRequestOptions(gitVersion.getVersion(), sourceIndex, - destinationIndex); + .setSourceIndices(sourceIndices) + .setDestIndex(destinationIndex) + .setMaxRetries(numRetries) + .setAbortOnVersionConflict(false) + .setSourceBatchSize(batchSize); + if (timeout != null) { + reindexRequest.setTimeout(timeout); + } + if (sourceFilterQuery != null) { + reindexRequest.setSourceQuery(sourceFilterQuery); + } + + RequestOptions requestOptions = ESUtils.buildReindexTaskRequestOptions(gitVersion.getVersion(), sourceIndices[0], + destinationIndex); TaskSubmissionResponse reindexTask = _searchClient.submitReindexTask(reindexRequest, requestOptions); return reindexTask.getTask(); } + private String submitReindex(String sourceIndex, String destinationIndex) throws IOException { + return submitReindex(new String[]{sourceIndex}, destinationIndex, 2500, null, null); + } + private Pair getDocumentCounts(String sourceIndex, String destinationIndex) throws Throwable { // Check whether reindex succeeded by comparing document count // There can be some delay between the reindex finishing and count being fully up to date, so try multiple times diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java index 8682eb9248032..0908d927cd40b 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java @@ -147,6 +147,17 @@ String deleteAspectValuesAsync( @Nonnull final Filter filter, @Nonnull final BatchWriteOperationsOptions options); + /** + * Reindex the index represented by entityName and aspect name, applying the filter + * @param entityName The name of the entity. + * @param aspectName The name of the aspect. + * @param filter A filter to be used when reindexing + * @param options Options to control reindex parameters + * @return The Job ID of the reindex operation + */ + String reindexAsync(@Nonnull String entityName, @Nonnull String aspectName, @Nonnull Filter filter, + @Nonnull BatchWriteOperationsOptions options); + /** * Rollback the Time-Series aspects associated with a particular runId. This is invoked as a part of an * ingestion rollback process. diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java index f92481ac5dc7e..f938e729d9f2a 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java @@ -62,6 +62,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -139,6 +140,10 @@ public void configure() { public List getReindexConfigs() { return _indexBuilders.getReindexConfigs(); } + public String reindexAsync(String index, @Nullable QueryBuilder filterQuery, BatchWriteOperationsOptions options) + throws Exception { + return _indexBuilders.reindexAsync(index, filterQuery, options); + } @Override public void reindexAll() { @@ -318,6 +323,19 @@ public String deleteAspectValuesAsync(@Nonnull String entityName, @Nonnull Strin } } + @Override + public String reindexAsync(@Nonnull String entityName, @Nonnull String aspectName, @Nonnull Filter filter, + @Nonnull BatchWriteOperationsOptions options) { + final String indexName = _indexConvention.getTimeseriesAspectIndexName(entityName, aspectName); + final BoolQueryBuilder filterQueryBuilder = ESUtils.buildFilterQuery(filter, true); + try { + return this.reindexAsync(indexName, filterQueryBuilder, options); + } catch (Exception e) { + log.error("Async reindex failed"); + throw new ESQueryException("Async reindex failed", e); + } + } + @Nonnull @Override public DeleteAspectValuesResult rollbackTimeseriesAspects(@Nonnull String runId) { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/indexbuilder/TimeseriesAspectIndexBuilders.java b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/indexbuilder/TimeseriesAspectIndexBuilders.java index 44db9696fa796..6c5dbf2582c05 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/indexbuilder/TimeseriesAspectIndexBuilders.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/indexbuilder/TimeseriesAspectIndexBuilders.java @@ -1,18 +1,23 @@ package com.linkedin.metadata.timeseries.elastic.indexbuilder; +import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ReindexConfig; import com.linkedin.metadata.shared.ElasticSearchIndexed; +import com.linkedin.metadata.timeseries.BatchWriteOperationsOptions; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import com.linkedin.util.Pair; +import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.elasticsearch.index.query.QueryBuilder; @Slf4j @@ -33,6 +38,30 @@ public void reindexAll() { } } + public String reindexAsync(String index, @Nullable QueryBuilder filterQuery, BatchWriteOperationsOptions options) + throws Exception { + Optional> entityAndAspect = _indexConvention.getEntityAndAspectName(index); + if (entityAndAspect.isEmpty()) { + throw new IllegalArgumentException("Could not extract entity and aspect from index " + index); + } + String entityName = entityAndAspect.get().getFirst(); + String aspectName = entityAndAspect.get().getSecond(); + EntitySpec entitySpec = _entityRegistry.getEntitySpec(entityName); + for (String aspect : entitySpec.getAspectSpecMap().keySet()) { + if (aspect.toLowerCase().equals(aspectName)) { + aspectName = aspect; + break; + } + } + if (!entitySpec.hasAspect(aspectName)) { + throw new IllegalArgumentException(String.format("Could not find aspect %s of entity %s", aspectName, entityName)); + } + ReindexConfig config = _indexBuilder.buildReindexState(index, + MappingsBuilder.getMappings(_entityRegistry.getEntitySpec(entityName).getAspectSpec(aspectName)), + Collections.emptyMap()); + return _indexBuilder.reindexInPlaceAsync(index, filterQuery, options, config); + } + @Override public List getReindexConfigs() { return _entityRegistry.getEntitySpecs().values().stream() diff --git a/metadata-service/restli-api/src/main/idl/com.linkedin.operations.operations.restspec.json b/metadata-service/restli-api/src/main/idl/com.linkedin.operations.operations.restspec.json index 35d3f21001258..958ec13b37fca 100644 --- a/metadata-service/restli-api/src/main/idl/com.linkedin.operations.operations.restspec.json +++ b/metadata-service/restli-api/src/main/idl/com.linkedin.operations.operations.restspec.json @@ -76,6 +76,14 @@ "name" : "timeoutSeconds", "type" : "long", "optional" : true + }, { + "name" : "forceDeleteByQuery", + "type" : "boolean", + "optional" : true + }, { + "name" : "forceReindex", + "type" : "boolean", + "optional" : true } ], "returns" : "string" } ], diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json index d6d5fec628113..f4c2d16f84747 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json @@ -3750,6 +3750,14 @@ "name" : "timeoutSeconds", "type" : "long", "optional" : true + }, { + "name" : "forceDeleteByQuery", + "type" : "boolean", + "optional" : true + }, { + "name" : "forceReindex", + "type" : "boolean", + "optional" : true } ], "returns" : "string" } ], diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/OperationsResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/OperationsResource.java index b98fd5c3cde19..17de9ceea35a3 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/OperationsResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/OperationsResource.java @@ -61,6 +61,8 @@ public class OperationsResource extends CollectionResourceTaskTemplate getTaskStatus( if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV)) && !isAuthorized(authentication, _authorizer, ImmutableList.of(PoliciesConfig.GET_ES_TASK_STATUS_PRIVILEGE), List.of(java.util.Optional.empty()))) { - throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED, "User is unauthorized to truncate timeseries index"); + throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED, "User is unauthorized to get ES task status"); } boolean taskSpecified = task != null; boolean nodeAndTaskIdSpecified = nodeId != null && taskId > 0; @@ -180,7 +182,9 @@ String executeTruncateTimeseriesAspect( @Nonnull Long endTimeMillis, @Nonnull Boolean dryRun, @Nullable Integer batchSize, - @Nullable Long timeoutSeconds + @Nullable Long timeoutSeconds, + @Nullable Boolean forceDeleteByQuery, + @Nullable Boolean forceReindex ) { Authentication authentication = AuthenticationContext.getAuthentication(); if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV)) @@ -189,7 +193,10 @@ String executeTruncateTimeseriesAspect( throw new RestLiServiceException(HttpStatus.S_401_UNAUTHORIZED, "User is unauthorized to truncate timeseries index"); } - // TODO(indy): Add optimization to perform a reindex if many documents will be truncated + if (forceDeleteByQuery != null && forceDeleteByQuery.equals(forceReindex)) { + return "please only set forceReindex OR forceDeleteByQuery flags"; + } + List criteria = new ArrayList<>(); criteria.add( QueryUtils.newCriterion("timestampMillis", String.valueOf(endTimeMillis), Condition.LESS_THAN_OR_EQUAL_TO)); @@ -199,6 +206,13 @@ String executeTruncateTimeseriesAspect( long totalNum = _timeseriesAspectService.countByFilter(entityType, aspectName, new Filter()); String deleteSummary = String.format("Delete %d out of %d rows (%.2f%%). ", numToDelete, totalNum, ((double) numToDelete) / totalNum * 100); + boolean reindex = !(forceDeleteByQuery != null && forceDeleteByQuery) && ((forceReindex != null && forceReindex) || numToDelete > (totalNum / 2)); + + if (reindex) { + deleteSummary += "Reindexing the aspect without the deleted records. "; + } else { + deleteSummary += "Issuing a delete by query request. "; + } if (dryRun) { deleteSummary += "This was a dry run. Run with dryRun = false to execute."; @@ -217,9 +231,21 @@ String executeTruncateTimeseriesAspect( options.setTimeoutSeconds(timeoutSeconds); } - String taskId = _timeseriesAspectService.deleteAspectValuesAsync(entityType, aspectName, filter, options); - log.info("delete by query request submitted with ID " + taskId); - return taskId; + if (reindex) { + // need to invert query to retain only the ones that do NOT meet the criterion from the count + List reindexCriteria = new ArrayList<>(); + reindexCriteria.add( + QueryUtils.newCriterion("timestampMillis", String.valueOf(endTimeMillis), Condition.GREATER_THAN)); + + final Filter reindexFilter = QueryUtils.getFilterFromCriteria(reindexCriteria); + String taskId = _timeseriesAspectService.reindexAsync(entityType, aspectName, reindexFilter, options); + log.info("reindex request submitted with ID " + taskId); + return taskId; + } else { + String taskId = _timeseriesAspectService.deleteAspectValuesAsync(entityType, aspectName, filter, options); + log.info("delete by query request submitted with ID " + taskId); + return taskId; + } } } @@ -232,10 +258,12 @@ public Task truncateTimeseriesAspect( @ActionParam(PARAM_END_TIME_MILLIS) @Nonnull Long endTimeMillis, @ActionParam(PARAM_IS_DRY_RUN) @Optional("true") @Nonnull Boolean dryRun, @ActionParam(PARAM_BATCH_SIZE) @Optional @Nullable Integer batchSize, - @ActionParam(PARAM_TIMEOUT_SECONDS) @Optional @Nullable Long timeoutSeconds + @ActionParam(PARAM_TIMEOUT_SECONDS) @Optional @Nullable Long timeoutSeconds, + @ActionParam(PARAM_FORCE_DELETE_BY_QUERY) @Optional @Nullable Boolean forceDeleteByQuery, + @ActionParam(PARAM_FORCE_REINDEX) @Optional @Nullable Boolean forceReindex ) { return RestliUtil.toTask(() -> - executeTruncateTimeseriesAspect(entityType, aspectName, endTimeMillis, dryRun, batchSize, timeoutSeconds), + executeTruncateTimeseriesAspect(entityType, aspectName, endTimeMillis, dryRun, batchSize, timeoutSeconds, forceDeleteByQuery, forceReindex), MetricRegistry.name(this.getClass(), "truncateTimeseriesAspect")); } } diff --git a/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/operations/OperationsResourceTest.java b/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/operations/OperationsResourceTest.java index c46785cf4d441..665bc3cfc277c 100644 --- a/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/operations/OperationsResourceTest.java +++ b/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/operations/OperationsResourceTest.java @@ -1,6 +1,8 @@ package com.linkedin.metadata.resources.operations; import com.linkedin.metadata.timeseries.TimeseriesAspectService; +import com.linkedin.util.Pair; +import java.util.List; import junit.framework.TestCase; import mock.MockTimeseriesAspectService; import org.testng.annotations.Test; @@ -9,19 +11,19 @@ public class OperationsResourceTest extends TestCase { private static final String TASK_ID = "taskId123"; - TimeseriesAspectService mockTimeseriesAspectService = new MockTimeseriesAspectService(); @Test public void testDryRun() { + TimeseriesAspectService mockTimeseriesAspectService = new MockTimeseriesAspectService(); String entityType = "dataset"; String aspectName = "datasetusagestatistics"; long endTimeMillis = 3000; OperationsResource testResource = new OperationsResource(mockTimeseriesAspectService); String output = testResource.executeTruncateTimeseriesAspect(entityType, aspectName, endTimeMillis, true, null, - null); + null, null, null); assertTrue(output.contains("This was a dry run")); output = testResource.executeTruncateTimeseriesAspect(entityType, aspectName, endTimeMillis, false, null, - null); + null, null, null); assertEquals(TASK_ID, output); } @@ -35,4 +37,63 @@ public void testIsTaskIdValid() { assertTrue(OperationsResource.isTaskIdValid("qhxGdzytQS-pQek8CwBCZg:54654")); assertTrue(OperationsResource.isTaskIdValid("qhxGdzytQSpQek8CwBCZg_:54654")); } + + @Test + public void testForceFlags() { + final String reindexTaskId = "REINDEX_TASK_ID"; + TimeseriesAspectService mockTimeseriesAspectServiceWouldDeleteByQuery = new MockTimeseriesAspectService(); + TimeseriesAspectService mockTimeseriesAspectServiceWouldReindex = new MockTimeseriesAspectService(30, 20, reindexTaskId); + String entityType = "dataset"; + String aspectName = "datasetusagestatistics"; + long endTimeMillis = 3000; + OperationsResource testResourceWouldReindex = new OperationsResource(mockTimeseriesAspectServiceWouldReindex); + OperationsResource testResourceWouldDeleteByQuery = new OperationsResource(mockTimeseriesAspectServiceWouldDeleteByQuery); + + String result = testResourceWouldReindex.executeTruncateTimeseriesAspect(entityType, aspectName, endTimeMillis, true, + null, null, true, true); + String errorIfFlagsAreIncompatable = "please only set forceReindex OR forceDeleteByQuery flags"; + assertEquals(errorIfFlagsAreIncompatable, result); + + + result = testResourceWouldReindex.executeTruncateTimeseriesAspect(entityType, aspectName, endTimeMillis, true, + null, null, false, false); + assertEquals(errorIfFlagsAreIncompatable, result); + + + List> + validOptionsNothingForced = List.of(Pair.of(null, null), Pair.of(null, false), Pair.of(false, null)); + for (Pair values : validOptionsNothingForced) { + String reindexResult = testResourceWouldReindex.executeTruncateTimeseriesAspect(entityType, aspectName, endTimeMillis, true, + null, null, values.getFirst(), values.getSecond()); + assertNotSame(errorIfFlagsAreIncompatable, reindexResult); + assertTrue(reindexResult.contains("Reindexing the aspect without the deleted records")); + String deleteResult = testResourceWouldDeleteByQuery.executeTruncateTimeseriesAspect(entityType, aspectName, endTimeMillis, true, + null, null, values.getFirst(), values.getSecond()); + assertNotSame(errorIfFlagsAreIncompatable, deleteResult); + assertTrue(deleteResult.contains("Issuing a delete by query request. ")); + } + + List> validOptionsForceDeleteByQuery = List.of(Pair.of(true, null), Pair.of(true, false)); + for (Pair values : validOptionsForceDeleteByQuery) { + String reindexResult = testResourceWouldReindex.executeTruncateTimeseriesAspect(entityType, aspectName, endTimeMillis, true, + null, null, values.getFirst(), values.getSecond()); + String deleteResult = testResourceWouldDeleteByQuery.executeTruncateTimeseriesAspect(entityType, aspectName, endTimeMillis, true, + null, null, values.getFirst(), values.getSecond()); + for (String res : List.of(reindexResult, deleteResult)) { + assertNotSame(errorIfFlagsAreIncompatable, res); + assertTrue(res.contains("Issuing a delete by query request. ")); + } + } + List> validOptionsForceReindex = List.of(Pair.of(null, true), Pair.of(false, true)); + for (Pair values : validOptionsForceReindex) { + String reindexResult = testResourceWouldReindex.executeTruncateTimeseriesAspect(entityType, aspectName, endTimeMillis, true, + null, null, values.getFirst(), values.getSecond()); + String deleteResult = testResourceWouldDeleteByQuery.executeTruncateTimeseriesAspect(entityType, aspectName, endTimeMillis, true, + null, null, values.getFirst(), values.getSecond()); + for (String res : List.of(reindexResult, deleteResult)) { + assertNotSame(errorIfFlagsAreIncompatable, res); + assertTrue(res.contains("Reindexing the aspect without the deleted records")); + } + } + } } \ No newline at end of file diff --git a/metadata-service/restli-servlet-impl/src/test/java/mock/MockTimeseriesAspectService.java b/metadata-service/restli-servlet-impl/src/test/java/mock/MockTimeseriesAspectService.java index 88f188f6fa7e2..81d2bbd88b3e6 100644 --- a/metadata-service/restli-servlet-impl/src/test/java/mock/MockTimeseriesAspectService.java +++ b/metadata-service/restli-servlet-impl/src/test/java/mock/MockTimeseriesAspectService.java @@ -81,6 +81,12 @@ public String deleteAspectValuesAsync(@Nonnull String entityName, @Nonnull Strin return _taskId; } + @Override + public String reindexAsync(@Nonnull String entityName, @Nonnull String aspectName, @Nonnull Filter filter, + @Nonnull BatchWriteOperationsOptions options) { + return _taskId; + } + @Nonnull @Override public DeleteAspectValuesResult rollbackTimeseriesAspects(@Nonnull String runId) { From 54c7aef1bcad94f68c5c511bd0f924c6e8798e84 Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Wed, 12 Jul 2023 20:56:13 +0200 Subject: [PATCH 132/222] feat(ingest/presto-on-hive): Extracting all the table properties from Hive Metastore (#8348) Co-authored-by: Pedro Silva Co-authored-by: Harshal Sheth --- metadata-ingestion/setup.py | 3 +- .../ingestion/source/sql/presto_on_hive.py | 93 +++++++++++++------ .../presto_on_hive_mces_golden_1.json | 65 ++++++++++--- .../presto_on_hive_mces_golden_2.json | 65 ++++++++++--- .../presto_on_hive_mces_golden_3.json | 65 ++++++++++--- .../presto_on_hive_mces_golden_4.json | 65 ++++++++++--- .../presto-on-hive/test_presto_on_hive.py | 2 +- .../tests/integration/s3/test_s3.py | 1 + 8 files changed, 279 insertions(+), 80 deletions(-) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 47a9d7f88e6ae..93925b3ec96f1 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -57,7 +57,8 @@ def get_long_description(): "click-spinner", "requests_file", "jsonref", - "jsonschema", + # jsonschema drops python 3.7 support in v4.18.0 + "jsonschema<=4.17.3", "ruamel.yaml", } diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py b/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py index 2afab226a8461..55d4ebc5ffa44 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py @@ -129,11 +129,6 @@ class PrestoOnHiveConfig(BasicSQLAlchemyConfig): description="Add the Presto catalog name (e.g. hive) to the generated dataset urns. `urn:li:dataset:(urn:li:dataPlatform:hive,hive.user.logging_events,PROD)` versus `urn:li:dataset:(urn:li:dataPlatform:hive,user.logging_events,PROD)`", ) - extra_properties: List[str] = Field( - default=[], - description="By default, the connector extracts a specific set of properties from the metastore tables with a sql query. Use this list of keys to provide additional properties that you would like to extract. You have to make sure the column name returned by the sql query is the same as the key you provide here.", - ) - enable_properties_merge: bool = Field( default=False, description="By default, the connector overwrites properties every time. Set this to True to enable merging of properties with what exists on the server.", @@ -171,25 +166,23 @@ class PrestoOnHiveSource(SQLAlchemySource): _TABLES_SQL_STATEMENT = """ SELECT source.* FROM - (SELECT t.TBL_ID, d.NAME as schema_name, t.TBL_NAME as table_name, t.TBL_TYPE as table_type, tp.PARAM_VALUE as description, + (SELECT t.TBL_ID, d.NAME as schema_name, t.TBL_NAME as table_name, t.TBL_TYPE as table_type, FROM_UNIXTIME(t.CREATE_TIME, '%Y-%m-%d') as create_date, p.PKEY_NAME as col_name, p.INTEGER_IDX as col_sort_order, p.PKEY_COMMENT as col_description, p.PKEY_TYPE as col_type, 1 as is_partition_col, s.LOCATION as table_location FROM TBLS t JOIN DBS d ON t.DB_ID = d.DB_ID JOIN SDS s ON t.SD_ID = s.SD_ID JOIN PARTITION_KEYS p ON t.TBL_ID = p.TBL_ID - LEFT JOIN TABLE_PARAMS tp ON (t.TBL_ID = tp.TBL_ID AND tp.PARAM_KEY='comment') WHERE t.TBL_TYPE IN ('EXTERNAL_TABLE', 'MANAGED_TABLE') {where_clause_suffix} UNION - SELECT t.TBL_ID, d.NAME as schema_name, t.TBL_NAME as table_name, t.TBL_TYPE as table_type, tp.PARAM_VALUE as description, + SELECT t.TBL_ID, d.NAME as schema_name, t.TBL_NAME as table_name, t.TBL_TYPE as table_type, FROM_UNIXTIME(t.CREATE_TIME, '%Y-%m-%d') as create_date, c.COLUMN_NAME as col_name, c.INTEGER_IDX as col_sort_order, c.COMMENT as col_description, c.TYPE_NAME as col_type, 0 as is_partition_col, s.LOCATION as table_location FROM TBLS t JOIN DBS d ON t.DB_ID = d.DB_ID JOIN SDS s ON t.SD_ID = s.SD_ID JOIN COLUMNS_V2 c ON s.CD_ID = c.CD_ID - LEFT JOIN TABLE_PARAMS tp ON (t.TBL_ID = tp.TBL_ID AND tp.PARAM_KEY='comment') WHERE t.TBL_TYPE IN ('EXTERNAL_TABLE', 'MANAGED_TABLE') {where_clause_suffix} ) source @@ -198,25 +191,23 @@ class PrestoOnHiveSource(SQLAlchemySource): _TABLES_POSTGRES_SQL_STATEMENT = """ SELECT source.* FROM - (SELECT t."TBL_ID" as tbl_id, d."NAME" as schema_name, t."TBL_NAME" as table_name, t."TBL_TYPE" as table_type, tp."PARAM_VALUE" as description, + (SELECT t."TBL_ID" as tbl_id, d."NAME" as schema_name, t."TBL_NAME" as table_name, t."TBL_TYPE" as table_type, to_char(to_timestamp(t."CREATE_TIME"), 'YYYY-MM-DD') as create_date, p."PKEY_NAME" as col_name, p."INTEGER_IDX" as col_sort_order, p."PKEY_COMMENT" as col_description, p."PKEY_TYPE" as col_type, 1 as is_partition_col, s."LOCATION" as table_location FROM "TBLS" t JOIN "DBS" d ON t."DB_ID" = d."DB_ID" JOIN "SDS" s ON t."SD_ID" = s."SD_ID" JOIN "PARTITION_KEYS" p ON t."TBL_ID" = p."TBL_ID" - LEFT JOIN "TABLE_PARAMS" tp ON (t."TBL_ID" = tp."TBL_ID" AND tp."PARAM_KEY"='comment') WHERE t."TBL_TYPE" IN ('EXTERNAL_TABLE', 'MANAGED_TABLE') {where_clause_suffix} UNION - SELECT t."TBL_ID" as tbl_id, d."NAME" as schema_name, t."TBL_NAME" as table_name, t."TBL_TYPE" as table_type, tp."PARAM_VALUE" as description, + SELECT t."TBL_ID" as tbl_id, d."NAME" as schema_name, t."TBL_NAME" as table_name, t."TBL_TYPE" as table_type, to_char(to_timestamp(t."CREATE_TIME"), 'YYYY-MM-DD') as create_date, c."COLUMN_NAME" as col_name, c."INTEGER_IDX" as col_sort_order, c."COMMENT" as col_description, c."TYPE_NAME" as col_type, 0 as is_partition_col, s."LOCATION" as table_location FROM "TBLS" t JOIN "DBS" d ON t."DB_ID" = d."DB_ID" JOIN "SDS" s ON t."SD_ID" = s."SD_ID" JOIN "COLUMNS_V2" c ON s."CD_ID" = c."CD_ID" - LEFT JOIN "TABLE_PARAMS" tp ON (t."TBL_ID" = tp."TBL_ID" AND tp."PARAM_KEY"='comment') WHERE t."TBL_TYPE" IN ('EXTERNAL_TABLE', 'MANAGED_TABLE') {where_clause_suffix} ) source @@ -273,6 +264,26 @@ class PrestoOnHiveSource(SQLAlchemySource): ORDER by tbl_id desc, col_sort_order asc; """ + _HIVE_PROPERTIES_SQL_STATEMENT = """ + SELECT d.NAME as schema_name, t.TBL_NAME as table_name, tp.PARAM_KEY, tp.PARAM_VALUE + FROM TABLE_PARAMS tp + JOIN TBLS t on t.TBL_ID = tp.TBL_ID + JOIN DBS d on d.DB_ID = t.DB_ID + WHERE 1 + {where_clause_suffix} + ORDER BY tp.TBL_ID desc; + """ + + _HIVE_PROPERTIES_POSTGRES_SQL_STATEMENT = """ + SELECT d."NAME" as schema_name, t."TBL_NAME" as table_name, tp."PARAM_KEY", tp."PARAM_VALUE" + FROM "TABLE_PARAMS" tp + JOIN "TBLS" t on t."TBL_ID" = tp."TBL_ID" + JOIN "DBS" d on d."DB_ID" = t."DB_ID" + WHERE 1 = 1 + {where_clause_suffix} + ORDER BY tp."TBL_ID" desc; + """ + _PRESTO_VIEW_PREFIX = "/* Presto View: " _PRESTO_VIEW_SUFFIX = " */" @@ -408,6 +419,31 @@ def get_default_ingestion_job_id(self) -> JobId: """ return JobId(self.config.ingestion_job_id) + def _get_table_properties( + self, db_name: str, scheme: str, where_clause_suffix: str + ) -> Dict[str, Dict[str, str]]: + statement: str = ( + PrestoOnHiveSource._HIVE_PROPERTIES_POSTGRES_SQL_STATEMENT.format( + where_clause_suffix=where_clause_suffix + ) + if "postgresql" in scheme + else PrestoOnHiveSource._HIVE_PROPERTIES_SQL_STATEMENT.format( + where_clause_suffix=where_clause_suffix + ) + ) + iter_res = self._alchemy_client.execute_query(statement) + table_properties: Dict[str, Dict[str, str]] = {} + for row in iter_res: + dataset_name = f"{row['schema_name']}.{row['table_name']}" + if self.config.include_catalog_name_in_ids: + dataset_name = f"{db_name}.{dataset_name}" + if row["PARAM_KEY"] and row["PARAM_VALUE"]: + table_properties.setdefault(dataset_name, {})[row["PARAM_KEY"]] = row[ + "PARAM_VALUE" + ] + + return table_properties + def loop_tables( self, inspector: Inspector, @@ -434,10 +470,17 @@ def loop_tables( ) ) + db_name = self.get_db_name(inspector) + + properties_cache = self._get_table_properties( + db_name=db_name, + scheme=sql_config.scheme, + where_clause_suffix=where_clause_suffix, + ) + iter_res = self._alchemy_client.execute_query(statement) for key, group in groupby(iter_res, self._get_table_key): - db_name = self.get_db_name(inspector) schema_name = ( f"{db_name}.{key.schema}" if self.config.include_catalog_name_in_ids @@ -488,11 +531,10 @@ def loop_tables( dataset_snapshot.aspects.append(schema_metadata) # add table properties - default_properties = ["create_date", "table_type", "table_location"] - properties: Dict[str, str] = {} - for prop in default_properties + self.config.extra_properties: - if prop in columns[-1]: - properties[prop] = str(columns[-1][prop]) or "" + properties: Dict[str, str] = properties_cache.get(dataset_name, {}) + properties["table_type"] = str(columns[-1]["table_type"] or "") + properties["table_location"] = str(columns[-1]["table_location"] or "") + properties["create_date"] = str(columns[-1]["create_date"] or "") par_columns: str = ", ".join( [c["col_name"] for c in columns if c["is_partition_col"]] @@ -500,10 +542,7 @@ def loop_tables( if par_columns != "": properties["partitioned_columns"] = par_columns - table_description = ( - columns[-1]["description"] if "description" in columns[-1] else "" - ) - + table_description = properties.get("comment") yield from self.add_hive_dataset_to_container( dataset_urn=dataset_urn, inspector=inspector, schema=key.schema ) @@ -514,9 +553,11 @@ def loop_tables( patch_builder: DatasetPatchBuilder = DatasetPatchBuilder( urn=dataset_snapshot.urn ) - patch_builder.set_display_name(key.table).set_description( - description=table_description - ) + patch_builder.set_display_name(key.table) + + if table_description: + patch_builder.set_description(description=table_description) + for prop, value in properties.items(): patch_builder.add_custom_property(key=prop, value=value) yield from [ diff --git a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json index f3439e0e1770b..0953b6738fdcb 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json +++ b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json @@ -264,9 +264,15 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", + "transient_lastDdlTime": "1688395014", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test", + "create_date": "2023-07-03" }, "name": "map_test", "tags": [] @@ -458,9 +464,15 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", + "transient_lastDdlTime": "1688395014", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test", + "create_date": "2023-07-03" }, "name": "union_test", "tags": [] @@ -644,9 +656,15 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", + "transient_lastDdlTime": "1688395014", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test", + "create_date": "2023-07-03" }, "name": "nested_struct_test", "tags": [] @@ -814,9 +832,17 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "comment": "This table has array of structs", + "numFiles": "1", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "transient_lastDdlTime": "1688395011", + "rawDataSize": "32", + "numRows": "1", + "totalSize": "33", + "another.comment": "This table has no partitions", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test", + "create_date": "2023-07-03" }, "name": "array_struct_test", "description": "This table has array of structs", @@ -979,9 +1005,15 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "transient_lastDdlTime": "1688395008", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test", + "create_date": "2023-07-03" }, "name": "struct_test", "tags": [] @@ -1113,9 +1145,15 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "transient_lastDdlTime": "1688395008", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore", + "create_date": "2023-07-03" }, "name": "_test_table_underscore", "tags": [] @@ -1260,9 +1298,10 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "transient_lastDdlTime": "1688395005", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", + "create_date": "2023-07-03", "partitioned_columns": "baz" }, "name": "pokes", diff --git a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json index 61f4e1e9be1e9..e5ac05f979368 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json +++ b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json @@ -264,9 +264,15 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", + "transient_lastDdlTime": "1688395014", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test", + "create_date": "2023-07-03" }, "name": "map_test", "tags": [] @@ -458,9 +464,15 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", + "transient_lastDdlTime": "1688395014", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test", + "create_date": "2023-07-03" }, "name": "union_test", "tags": [] @@ -644,9 +656,15 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", + "transient_lastDdlTime": "1688395014", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test", + "create_date": "2023-07-03" }, "name": "nested_struct_test", "tags": [] @@ -814,9 +832,17 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "comment": "This table has array of structs", + "numFiles": "1", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "transient_lastDdlTime": "1688395011", + "rawDataSize": "32", + "numRows": "1", + "totalSize": "33", + "another.comment": "This table has no partitions", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test", + "create_date": "2023-07-03" }, "name": "array_struct_test", "description": "This table has array of structs", @@ -979,9 +1005,15 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "transient_lastDdlTime": "1688395008", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test", + "create_date": "2023-07-03" }, "name": "struct_test", "tags": [] @@ -1113,9 +1145,15 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "transient_lastDdlTime": "1688395008", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore", + "create_date": "2023-07-03" }, "name": "_test_table_underscore", "tags": [] @@ -1260,9 +1298,10 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "transient_lastDdlTime": "1688395005", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", + "create_date": "2023-07-03", "partitioned_columns": "baz" }, "name": "pokes", diff --git a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_3.json b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_3.json index b1b3bf93d6ed3..a9cf64c0a43b4 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_3.json +++ b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_3.json @@ -264,9 +264,15 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", + "transient_lastDdlTime": "1688395014", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test", + "create_date": "2023-07-03" }, "name": "map_test", "tags": [] @@ -458,9 +464,15 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", + "transient_lastDdlTime": "1688395014", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test", + "create_date": "2023-07-03" }, "name": "union_test", "tags": [] @@ -644,9 +656,15 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", + "transient_lastDdlTime": "1688395014", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test", + "create_date": "2023-07-03" }, "name": "nested_struct_test", "tags": [] @@ -814,9 +832,17 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "comment": "This table has array of structs", + "numFiles": "1", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "transient_lastDdlTime": "1688395011", + "rawDataSize": "32", + "numRows": "1", + "totalSize": "33", + "another.comment": "This table has no partitions", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test", + "create_date": "2023-07-03" }, "name": "array_struct_test", "description": "This table has array of structs", @@ -979,9 +1005,15 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "transient_lastDdlTime": "1688395008", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test", + "create_date": "2023-07-03" }, "name": "struct_test", "tags": [] @@ -1113,9 +1145,15 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "transient_lastDdlTime": "1688395008", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore", + "create_date": "2023-07-03" }, "name": "_test_table_underscore", "tags": [] @@ -1260,9 +1298,10 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "transient_lastDdlTime": "1688395005", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", + "create_date": "2023-07-03", "partitioned_columns": "baz" }, "name": "pokes", diff --git a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_4.json b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_4.json index 5a41a32cbf0e2..a050f418371c6 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_4.json +++ b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_4.json @@ -264,9 +264,15 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", + "transient_lastDdlTime": "1688395014", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test", + "create_date": "2023-07-03" }, "name": "map_test", "tags": [] @@ -458,9 +464,15 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", + "transient_lastDdlTime": "1688395014", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test", + "create_date": "2023-07-03" }, "name": "union_test", "tags": [] @@ -644,9 +656,15 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", + "transient_lastDdlTime": "1688395014", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test", + "create_date": "2023-07-03" }, "name": "nested_struct_test", "tags": [] @@ -814,9 +832,17 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "comment": "This table has array of structs", + "numFiles": "1", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "transient_lastDdlTime": "1688395011", + "rawDataSize": "32", + "numRows": "1", + "totalSize": "33", + "another.comment": "This table has no partitions", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test", + "create_date": "2023-07-03" }, "name": "array_struct_test", "description": "This table has array of structs", @@ -979,9 +1005,15 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "transient_lastDdlTime": "1688395008", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test", + "create_date": "2023-07-03" }, "name": "struct_test", "tags": [] @@ -1113,9 +1145,15 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "transient_lastDdlTime": "1688395008", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore", + "create_date": "2023-07-03" }, "name": "_test_table_underscore", "tags": [] @@ -1260,9 +1298,10 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2023-07-05", + "transient_lastDdlTime": "1688395005", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", + "create_date": "2023-07-03", "partitioned_columns": "baz" }, "name": "pokes", diff --git a/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py b/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py index 5ba849d645a77..8828f0a92e40d 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py +++ b/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py @@ -124,7 +124,7 @@ def test_presto_on_hive_ingest( golden_path=test_resources_dir / f"presto_on_hive_mces_golden{test_suffix}.json", ignore_paths=[ - r"root\[\d+\]\['proposedSnapshot'\]\['com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot'\]\['aspects'\]\[\d+\]\['com.linkedin.pegasus2avro.dataset.DatasetProperties'\]\['customProperties'\]\['transient_lastddltime'\]", + r"root\[\d+\]\['proposedSnapshot'\]\['com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot'\]\['aspects'\]\[\d+\]\['com.linkedin.pegasus2avro.dataset.DatasetProperties'\]\['customProperties'\]\['transient_lastDdlTime'\]", r"root\[\d+\]\['proposedSnapshot'\]\['com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot'\]\['aspects'\]\[\d+\]\['com.linkedin.pegasus2avro.dataset.DatasetProperties'\]\['customProperties'\]\['numfiles'\]", r"root\[\d+\]\['proposedSnapshot'\]\['com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot'\]\['aspects'\]\[\d+\]\['com.linkedin.pegasus2avro.dataset.DatasetProperties'\]\['customProperties'\]\['totalsize'\]", r"root\[\d+\]\['proposedSnapshot'\]\['com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot'\]\['aspects'\]\[\d+\]\['com.linkedin.pegasus2avro.dataset.DatasetProperties'\]\['customProperties'\]\['create_date'\]", diff --git a/metadata-ingestion/tests/integration/s3/test_s3.py b/metadata-ingestion/tests/integration/s3/test_s3.py index 645593b1a46bb..6202fa5509beb 100644 --- a/metadata-ingestion/tests/integration/s3/test_s3.py +++ b/metadata-ingestion/tests/integration/s3/test_s3.py @@ -127,6 +127,7 @@ def test_data_lake_s3_ingest( def test_data_lake_local_ingest( pytestconfig, touch_local_files, source_file, tmp_path, mock_time ): + os.environ["SPARK_VERSION"] = "3.0.3" test_resources_dir = pytestconfig.rootpath / "tests/integration/s3/" f = open(os.path.join(SOURCE_FILES_PATH, source_file)) source = json.load(f) From 61a1f5775cc85e9c05414bc0af1a758f53976ef0 Mon Sep 17 00:00:00 2001 From: Indy Prentice Date: Wed, 12 Jul 2023 17:29:03 -0300 Subject: [PATCH 133/222] feat(openapi): Add openapi endpoint for getting task status (#8391) Co-authored-by: Indy Prentice --- .../elastic/OperationsController.java | 83 +++++++++++++++++++ .../openapi/util/ElasticsearchUtils.java | 15 ++++ 2 files changed, 98 insertions(+) create mode 100644 metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/operations/elastic/OperationsController.java create mode 100644 metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/ElasticsearchUtils.java diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/operations/elastic/OperationsController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/operations/elastic/OperationsController.java new file mode 100644 index 0000000000000..7910982a63133 --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/operations/elastic/OperationsController.java @@ -0,0 +1,83 @@ +package io.datahubproject.openapi.operations.elastic; + +import com.datahub.authentication.Authentication; +import com.datahub.authentication.AuthenticationContext; +import com.datahub.authorization.AuthUtil; +import com.datahub.authorization.AuthorizerChain; +import com.datahub.authorization.ConjunctivePrivilegeGroup; +import com.datahub.authorization.DisjunctivePrivilegeGroup; +import io.datahubproject.openapi.util.ElasticsearchUtils; +import com.google.common.collect.ImmutableList; +import com.linkedin.metadata.authorization.PoliciesConfig; +import com.linkedin.metadata.systemmetadata.SystemMetadataService; +import io.swagger.v3.oas.annotations.tags.Tag; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.elasticsearch.client.tasks.GetTaskResponse; +import org.json.JSONObject; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.propertyeditors.StringArrayPropertyEditor; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.WebDataBinder; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.InitBinder; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + + +@RestController +@RequestMapping("/operations/elasticSearch") +@Slf4j +@Tag(name = "ElasticSearchOperations", description = "An API for managing your elasticsearch instance") +public class OperationsController { + private final AuthorizerChain _authorizerChain; + + @Value("${authorization.restApiAuthorization:false}") + private boolean restApiAuthorizationEnabled; + + @Autowired + @Qualifier("elasticSearchSystemMetadataService") + private SystemMetadataService _systemMetadataService; + + public OperationsController(AuthorizerChain authorizerChain) { + _authorizerChain = authorizerChain; + } + + @InitBinder + public void initBinder(WebDataBinder binder) { + binder.registerCustomEditor(String[].class, new StringArrayPropertyEditor(null)); + } + @GetMapping(path = "/getTaskStatus", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity getTaskStatus(String task) { + Authentication authentication = AuthenticationContext.getAuthentication(); + String actorUrnStr = authentication.getActor().toUrnStr(); + DisjunctivePrivilegeGroup orGroup = new DisjunctivePrivilegeGroup(ImmutableList.of(new ConjunctivePrivilegeGroup( + ImmutableList.of(PoliciesConfig.GET_ES_TASK_STATUS_PRIVILEGE.getType()) + ))); + if (restApiAuthorizationEnabled && !AuthUtil.isAuthorizedForResources(_authorizerChain, actorUrnStr, List.of(java.util.Optional.empty()), orGroup)) { + return ResponseEntity.status(HttpStatus.FORBIDDEN).body( + String.format(actorUrnStr + " is not authorized to get ElasticSearch task status")); + } + if (!ElasticsearchUtils.isTaskIdValid(task)) { + return ResponseEntity.status(HttpStatus.BAD_REQUEST).body( + String.format("Task ID should be in the form nodeId:taskId e.g. aB1cdEf2GHI-JKLMnoPQr3:123456 (got %s)", task)); + } + String nodeIdToQuery = task.split(":")[0]; + long taskIdToQuery = Long.parseLong(task.split(":")[1]); + java.util.Optional res = _systemMetadataService.getTaskStatus(nodeIdToQuery, taskIdToQuery); + if (res.isEmpty()) { + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(String.format("Could not get task status for %s:%d", nodeIdToQuery, taskIdToQuery)); + } + GetTaskResponse resp = res.get(); + JSONObject j = new JSONObject(); + j.put("completed", resp.isCompleted()); + j.put("taskId", res.get().getTaskInfo().getTaskId()); + j.put("status", res.get().getTaskInfo().getStatus()); + j.put("runTimeNanos", res.get().getTaskInfo().getRunningTimeNanos()); + return ResponseEntity.ok(j.toString()); + } +} \ No newline at end of file diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/ElasticsearchUtils.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/ElasticsearchUtils.java new file mode 100644 index 0000000000000..9ef14eefc429b --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/ElasticsearchUtils.java @@ -0,0 +1,15 @@ +package io.datahubproject.openapi.util; + +public class ElasticsearchUtils { + private ElasticsearchUtils() { } + public static boolean isTaskIdValid(String task) { + if (task.matches("^[a-zA-Z0-9-_]+:[0-9]+$")) { + try { + return Long.parseLong(task.split(":")[1]) != 0; + } catch (NumberFormatException e) { + return false; + } + } + return false; + } +} From 64bcc132a3d5f39d82553ca81857a8f73e88306b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90=E1=BA=B7ng=20Minh=20D=C5=A9ng?= Date: Thu, 13 Jul 2023 06:37:36 +0700 Subject: [PATCH 134/222] feat(ingest/airflow): able to set `platform_instance` in `Dataset` (#8313) Co-authored-by: Andrew Sikowitz --- metadata-ingestion/src/datahub_provider/entities.py | 9 ++++++++- .../example_dags/lineage_backend_demo.py | 9 +++++++-- .../example_dags/lineage_emission_dag.py | 12 +++++++++--- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/metadata-ingestion/src/datahub_provider/entities.py b/metadata-ingestion/src/datahub_provider/entities.py index d54db1c286e30..bfccc2f22eeb8 100644 --- a/metadata-ingestion/src/datahub_provider/entities.py +++ b/metadata-ingestion/src/datahub_provider/entities.py @@ -1,4 +1,5 @@ from abc import abstractmethod +from typing import Optional import attr @@ -18,10 +19,16 @@ class Dataset(_Entity): platform: str name: str env: str = builder.DEFAULT_ENV + platform_instance: Optional[str] = None @property def urn(self): - return builder.make_dataset_urn(self.platform, self.name, self.env) + return builder.make_dataset_urn_with_platform_instance( + platform=self.platform, + name=self.name, + platform_instance=self.platform_instance, + env=self.env, + ) @attr.s(str=True) diff --git a/metadata-ingestion/src/datahub_provider/example_dags/lineage_backend_demo.py b/metadata-ingestion/src/datahub_provider/example_dags/lineage_backend_demo.py index e04159f388f4f..95b594e4052a5 100644 --- a/metadata-ingestion/src/datahub_provider/example_dags/lineage_backend_demo.py +++ b/metadata-ingestion/src/datahub_provider/example_dags/lineage_backend_demo.py @@ -34,8 +34,13 @@ dag=dag, bash_command="echo 'This is where you might run your data tooling.'", inlets=[ - Dataset("snowflake", "mydb.schema.tableA"), - Dataset("snowflake", "mydb.schema.tableB", "DEV"), + Dataset(platform="snowflake", name="mydb.schema.tableA"), + Dataset(platform="snowflake", name="mydb.schema.tableB", env="DEV"), + Dataset( + platform="snowflake", + name="mydb.schema.tableC", + platform_instance="cloud", + ), # You can also put dataset URNs in the inlets/outlets lists. Urn( "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)" diff --git a/metadata-ingestion/src/datahub_provider/example_dags/lineage_emission_dag.py b/metadata-ingestion/src/datahub_provider/example_dags/lineage_emission_dag.py index c77ae3f00aab5..153464246cef7 100644 --- a/metadata-ingestion/src/datahub_provider/example_dags/lineage_emission_dag.py +++ b/metadata-ingestion/src/datahub_provider/example_dags/lineage_emission_dag.py @@ -57,11 +57,17 @@ mces=[ builder.make_lineage_mce( upstream_urns=[ - builder.make_dataset_urn("snowflake", "mydb.schema.tableA"), - builder.make_dataset_urn("snowflake", "mydb.schema.tableB"), + builder.make_dataset_urn( + platform="snowflake", name="mydb.schema.tableA" + ), + builder.make_dataset_urn_with_platform_instance( + platform="snowflake", + name="mydb.schema.tableB", + platform_instance="cloud", + ), ], downstream_urn=builder.make_dataset_urn( - "snowflake", "mydb.schema.tableC" + platform="snowflake", name="mydb.schema.tableC", env="DEV" ), ) ], From 851c5bab9e18f25572dac5493c5855e657a6fff3 Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Thu, 13 Jul 2023 02:41:07 -0400 Subject: [PATCH 135/222] test(ingest/minio): Configure delta lake minio tests for arm64 (#8364) Co-authored-by: Tamas Nemeth --- metadata-ingestion/.gitignore | 1 - .../integration/delta_lake/docker-compose.yml | 12 ++++ .../delta_lake/minio/kill_minio.sh | 3 - .../delta_lake/minio/setup_minio.sh | 6 -- .../_delta_log/00000000000000000000.json | 4 -- .../_delta_log/00000000000000000001.json | 2 - .../_delta_log/00000000000000000002.json | 2 - .../_delta_log/00000000000000000003.json | 2 - .../_delta_log/00000000000000000004.json | 2 - .../delta_lake/test_delta_lake_minio.py | 62 +++++++++++-------- 10 files changed, 48 insertions(+), 48 deletions(-) create mode 100644 metadata-ingestion/tests/integration/delta_lake/docker-compose.yml delete mode 100755 metadata-ingestion/tests/integration/delta_lake/minio/kill_minio.sh delete mode 100755 metadata-ingestion/tests/integration/delta_lake/minio/setup_minio.sh delete mode 100644 metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000000.json delete mode 100644 metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000001.json delete mode 100644 metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000002.json delete mode 100644 metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000003.json delete mode 100644 metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000004.json diff --git a/metadata-ingestion/.gitignore b/metadata-ingestion/.gitignore index 4668c8adbbf2e..673c8e0995872 100644 --- a/metadata-ingestion/.gitignore +++ b/metadata-ingestion/.gitignore @@ -6,7 +6,6 @@ pvenv36/ /venv*/ bq_credentials.json junit.*.xml -tests/integrations/delta_lake/minio/minio /tmp *.bak diff --git a/metadata-ingestion/tests/integration/delta_lake/docker-compose.yml b/metadata-ingestion/tests/integration/delta_lake/docker-compose.yml new file mode 100644 index 0000000000000..0b27218d23b96 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/docker-compose.yml @@ -0,0 +1,12 @@ +version: "3" +services: + minio: + image: minio/minio:RELEASE.2023-07-07T07-13-57Z + container_name: "minio_test" + environment: + MINIO_ROOT_USER: "miniouser" + MINIO_ROOT_PASSWORD: "miniopassword" + ports: + - 9000:9000 # S3 API + - 9001:9001 # Web UI + command: server /data --console-address ":9001" diff --git a/metadata-ingestion/tests/integration/delta_lake/minio/kill_minio.sh b/metadata-ingestion/tests/integration/delta_lake/minio/kill_minio.sh deleted file mode 100755 index 338119fe13d84..0000000000000 --- a/metadata-ingestion/tests/integration/delta_lake/minio/kill_minio.sh +++ /dev/null @@ -1,3 +0,0 @@ -kill -9 `cat minio_pid.txt` -rm ./tests/integrations/delta_lake/minio/minio_pid.txt -rm -rf ./tests/integrations/delta_lake/minio/data \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/delta_lake/minio/setup_minio.sh b/metadata-ingestion/tests/integration/delta_lake/minio/setup_minio.sh deleted file mode 100755 index cd5cad6f0afee..0000000000000 --- a/metadata-ingestion/tests/integration/delta_lake/minio/setup_minio.sh +++ /dev/null @@ -1,6 +0,0 @@ -#ref https://hub.docker.com/r/minio/minio/ - -wget https://dl.min.io/server/minio/release/linux-amd64/minio -P ./tests/integrations/delta_lake/minio/ -chmod +x ./tests/integrations/delta_lake/minio/minio -nohup ./tests/integrations/delta_lake/minio/minio server ./tests/integrations/delta_lake/minio/data > temp.log 2>&1 & -echo $! > ./tests/integrations/delta_lake/minio/minio_pid.txt diff --git a/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000000.json b/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000000.json deleted file mode 100644 index c344027562aa8..0000000000000 --- a/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000000.json +++ /dev/null @@ -1,4 +0,0 @@ -{"commitInfo":{"timestamp":1655724801453,"operation":"WRITE","operationParameters":{},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} -{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} -{"metaData":{"id":"468553a9-3b52-4e50-b855-e64d2d070cb8","name":"test-table","description":"test delta table","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"foo\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"bar\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"zip\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["foo","bar"],"configuration":{},"createdTime":1655724801057}} -{"add":{"path":"0","partitionValues":{"bar":"0","foo":"0"},"size":100,"modificationTime":1655724801375,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000001.json b/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000001.json deleted file mode 100644 index 181b31f71db52..0000000000000 --- a/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000001.json +++ /dev/null @@ -1,2 +0,0 @@ -{"commitInfo":{"timestamp":1655724802209,"operation":"WRITE","operationParameters":{},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} -{"add":{"path":"1","partitionValues":{"bar":"1","foo":"1"},"size":100,"modificationTime":1655724802208,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000002.json b/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000002.json deleted file mode 100644 index 007ed0efdd831..0000000000000 --- a/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000002.json +++ /dev/null @@ -1,2 +0,0 @@ -{"commitInfo":{"timestamp":1655724802227,"operation":"WRITE","operationParameters":{},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} -{"add":{"path":"2","partitionValues":{"bar":"0","foo":"2"},"size":100,"modificationTime":1655724802226,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000003.json b/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000003.json deleted file mode 100644 index 8ed5a1c9ec6b1..0000000000000 --- a/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000003.json +++ /dev/null @@ -1,2 +0,0 @@ -{"commitInfo":{"timestamp":1655724802244,"operation":"WRITE","operationParameters":{},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} -{"add":{"path":"3","partitionValues":{"bar":"1","foo":"0"},"size":100,"modificationTime":1655724802243,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000004.json b/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000004.json deleted file mode 100644 index 23423b22f9fdc..0000000000000 --- a/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000004.json +++ /dev/null @@ -1,2 +0,0 @@ -{"commitInfo":{"timestamp":1655724802264,"operation":"WRITE","operationParameters":{},"readVersion":3,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} -{"add":{"path":"4","partitionValues":{"bar":"0","foo":"1"},"size":100,"modificationTime":1655724802263,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_delta_lake_minio.py b/metadata-ingestion/tests/integration/delta_lake/test_delta_lake_minio.py index ac13edfa8c6d4..36ec1d317fec4 100644 --- a/metadata-ingestion/tests/integration/delta_lake/test_delta_lake_minio.py +++ b/metadata-ingestion/tests/integration/delta_lake/test_delta_lake_minio.py @@ -2,47 +2,58 @@ import subprocess import boto3 +import freezegun import pytest from datahub.ingestion.run.pipeline import Pipeline from tests.test_helpers import mce_helpers +from tests.test_helpers.docker_helpers import wait_for_port FROZEN_TIME = "2020-04-14 07:00:00" +MINIO_PORT = 9000 -@pytest.fixture(scope="module", autouse=True) -def minio_startup(): - cmd = "./tests/integration/delta_lake/minio/setup_minio.sh" - ret = subprocess.run( - cmd, - shell=True, - ) - assert ret.returncode == 0 - yield +def is_minio_up(container_name: str) -> bool: + """A cheap way to figure out if postgres is responsive on a container""" - # Shutdown minio server - cmd = "./tests/integration/delta_lake/minio/kill_minio.sh" + cmd = f"docker logs {container_name} 2>&1 | grep '1 Online'" ret = subprocess.run( cmd, shell=True, ) - assert ret.returncode == 0 + return ret.returncode == 0 -@pytest.fixture(scope="module", autouse=True) -def bucket_name(): - return "my-test-bucket" +@pytest.fixture(scope="module") +def test_resources_dir(pytestconfig): + return pytestconfig.rootpath / "tests/integration/delta_lake" + + +@pytest.fixture(scope="module") +def minio_runner(docker_compose_runner, pytestconfig, test_resources_dir): + container_name = "minio_test" + with docker_compose_runner( + test_resources_dir / "docker-compose.yml", container_name + ) as docker_services: + wait_for_port( + docker_services, + container_name, + MINIO_PORT, + timeout=120, + checker=lambda: is_minio_up(container_name), + ) + yield docker_services @pytest.fixture(scope="module", autouse=True) -def s3_bkt(bucket_name, minio_startup): +def s3_bkt(minio_runner): s3 = boto3.resource( "s3", - endpoint_url="http://localhost:9000", - aws_access_key_id="minioadmin", - aws_secret_access_key="minioadmin", + endpoint_url=f"http://localhost:{MINIO_PORT}", + aws_access_key_id="miniouser", + aws_secret_access_key="miniopassword", ) - bkt = s3.Bucket(bucket_name) + bkt = s3.Bucket("my-test-bucket") bkt.create() return bkt @@ -62,9 +73,8 @@ def populate_minio(pytestconfig, s3_bkt): @pytest.mark.slow_integration -def test_delta_lake_ingest(pytestconfig, tmp_path, mock_time): - test_resources_dir = pytestconfig.rootpath / "tests/integration/delta_lake/" - +@freezegun.freeze_time("2023-01-01 00:00:00+00:00") +def test_delta_lake_ingest(pytestconfig, tmp_path, test_resources_dir): # Run the metadata ingestion pipeline. pipeline = Pipeline.create( { @@ -76,9 +86,9 @@ def test_delta_lake_ingest(pytestconfig, tmp_path, mock_time): "base_path": "s3://my-test-bucket/delta_tables/sales", "s3": { "aws_config": { - "aws_access_key_id": "minioadmin", - "aws_secret_access_key": "minioadmin", - "aws_endpoint_url": "http://localhost:9000", + "aws_access_key_id": "miniouser", + "aws_secret_access_key": "miniopassword", + "aws_endpoint_url": f"http://localhost:{MINIO_PORT}", "aws_region": "us-east-1", }, }, From 6ce157d0ffcf1829ac7f2e31b82bc6938b7afa2f Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Thu, 13 Jul 2023 03:27:52 -0400 Subject: [PATCH 136/222] docs(ingest): Add warning for Python 3.7 deprecation (#8411) --- metadata-ingestion/src/datahub/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/metadata-ingestion/src/datahub/__init__.py b/metadata-ingestion/src/datahub/__init__.py index cbe5eb47d5033..3ac3efefc14f0 100644 --- a/metadata-ingestion/src/datahub/__init__.py +++ b/metadata-ingestion/src/datahub/__init__.py @@ -23,3 +23,10 @@ def nice_version_name() -> str: FutureWarning, stacklevel=2, ) +elif sys.version_info < (3, 8): + warnings.warn( + "DataHub will require Python 3.8 or newer soon. " + "Please upgrade your Python version to continue using DataHub.", + FutureWarning, + stacklevel=2, + ) From 60b962367576ba06d669ef8d0b9b1a50a011c806 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Thu, 13 Jul 2023 12:58:19 +0530 Subject: [PATCH 137/222] fix(ingest/tableau): graceful handling of get all datasources failure (#8406) --- .../src/datahub/ingestion/source/tableau.py | 132 ++++++++++++++---- .../tableau/test_tableau_ingest.py | 51 ++++++- 2 files changed, 145 insertions(+), 38 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau.py index 7d1ead5e46b5a..5ad39425c3f73 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau.py @@ -411,6 +411,15 @@ def update_table( self.num_cols = num_tbl_cols +class TableauSourceReport(StaleEntityRemovalSourceReport): + get_all_datasources_query_failed: bool = False + num_get_datasource_query_failures: int = 0 + num_datasource_field_skipped_no_name: int = 0 + num_csql_field_skipped_no_name: int = 0 + num_table_field_skipped_no_name: int = 0 + num_upstream_table_skipped_no_name: int = 0 + + @platform_name("Tableau") @config_class(TableauConfig) @support_status(SupportStatus.CERTIFIED) @@ -442,7 +451,7 @@ def __init__( super().__init__(config, ctx) self.config: TableauConfig = config - self.report: StaleEntityRemovalSourceReport = StaleEntityRemovalSourceReport() + self.report: TableauSourceReport = TableauSourceReport() self.server: Optional[Server] = None self.database_tables: Dict[str, DatabaseTable] = {} self.tableau_stat_registry: Dict[str, UsageStat] = {} @@ -595,14 +604,20 @@ def _init_datasource_registry(self) -> None: if self.server is None: return - for ds in TSC.Pager(self.server.datasources): - if ds.project_id not in self.tableau_project_registry: - logger.debug( - f"project id ({ds.project_id}) of datasource {ds.name} is not present in project " - f"registry" - ) - continue - self.datasource_project_map[ds.id] = ds.project_id + try: + for ds in TSC.Pager(self.server.datasources): + if ds.project_id not in self.tableau_project_registry: + logger.debug( + f"project id ({ds.project_id}) of datasource {ds.name} is not present in project " + f"registry" + ) + continue + self.datasource_project_map[ds.id] = ds.project_id + self.report.get_all_datasources_query_failed = False + except Exception as e: + self.report.get_all_datasources_query_failed = True + logger.info(f"Get all datasources query failed due to error {e}") + logger.debug("Error stack trace", exc_info=True) def _init_workbook_registry(self) -> None: if self.server is None: @@ -956,6 +971,7 @@ def get_upstream_tables(self, tables, datasource_name, browse_path, is_custom_sq ) continue elif table[tableau_constant.NAME] is None: + self.report.num_upstream_table_skipped_no_name += 1 logger.warning( f"Skipping upstream table {table[tableau_constant.ID]} from lineage since its name is none: {table}" ) @@ -1263,6 +1279,7 @@ def get_schema_metadata_for_custom_sql( # Datasource fields if field.get(tableau_constant.NAME) is None: + self.report.num_csql_field_skipped_no_name += 1 logger.warning( f"Skipping field {field[tableau_constant.ID]} from schema since its name is none" ) @@ -1290,6 +1307,16 @@ def get_schema_metadata_for_custom_sql( return schema_metadata def _get_published_datasource_project_luid(self, ds): + # This is fallback in case "get all datasources" query fails for some reason. + # It is possible due to https://github.com/tableau/server-client-python/issues/1210 + if ( + ds.get(tableau_constant.LUID) + and ds[tableau_constant.LUID] not in self.datasource_project_map.keys() + and self.report.get_all_datasources_query_failed + ): + # Query and update self.datasource_project_map with luid + self._query_published_datasource_for_project_luid(ds) + if ( ds.get(tableau_constant.LUID) and ds[tableau_constant.LUID] in self.datasource_project_map.keys() @@ -1304,6 +1331,30 @@ def _get_published_datasource_project_luid(self, ds): return None + def _query_published_datasource_for_project_luid(self, ds: dict) -> None: + if self.server is None: + return + + try: + logger.debug( + f"published datasource {ds.get(tableau_constant.NAME)} project_luid not found." + f" Running get datasource query for {ds[tableau_constant.LUID]}" + ) + ds_result = self.server.datasources.get_by_id(ds[tableau_constant.LUID]) + if ds_result.project_id not in self.tableau_project_registry: + logger.debug( + f"project id ({ds_result.project_id}) of datasource {ds_result.name} is not present in project " + f"registry" + ) + else: + self.datasource_project_map[ds_result.id] = ds_result.project_id + except Exception as e: + self.report.num_get_datasource_query_failures += 1 + logger.warning( + f"Failed to get datasource project_luid for {ds[tableau_constant.LUID]} due to error {e}" + ) + logger.debug("Error stack trace", exc_info=True) + def _get_workbook_project_luid(self, wb): if wb.get(tableau_constant.LUID) and self.workbook_project_map.get( wb[tableau_constant.LUID] @@ -1451,6 +1502,7 @@ def _get_schema_metadata_for_datasource( # check datasource - custom sql relations from a field being referenced self._track_custom_sql_ids(field) if field.get(tableau_constant.NAME) is None: + self.report.num_upstream_table_skipped_no_name += 1 logger.warning( f"Skipping field {field[tableau_constant.ID]} from schema since its name is none" ) @@ -1619,29 +1671,38 @@ def emit_datasource( ), ) - if ( - is_embedded_ds and workbook is not None - ): # It is embedded then parent is container is workbook - yield from add_entity_to_container( - self.gen_workbook_key(workbook), - tableau_constant.DATASET, - dataset_snapshot.urn, - ) - elif ( - datasource.get(tableau_constant.LUID) - and datasource[tableau_constant.LUID] in self.datasource_project_map.keys() - ): # It is published datasource and hence parent container is project + container_key = self._get_datasource_container_key( + datasource, workbook, is_embedded_ds + ) + if container_key is not None: yield from add_entity_to_container( - self.gen_project_key( - self.datasource_project_map[datasource[tableau_constant.LUID]] - ), + container_key, tableau_constant.DATASET, dataset_snapshot.urn, ) + + def _get_datasource_container_key(self, datasource, workbook, is_embedded_ds): + container_key: Optional[PlatformKey] = None + if is_embedded_ds: # It is embedded then parent is container is workbook + if workbook is not None: + container_key = self.gen_workbook_key(workbook) + else: + logger.warning( + f"Parent container not set for embedded datasource {datasource[tableau_constant.ID]}" + ) else: - logger.warning( - f"Parent container not set for datasource {datasource[tableau_constant.ID]}" + parent_project_luid = self._get_published_datasource_project_luid( + datasource ) + # It is published datasource and hence parent container is project + if parent_project_luid is not None: + container_key = self.gen_project_key(parent_project_luid) + else: + logger.warning( + f"Parent container not set for published datasource {datasource[tableau_constant.ID]}" + ) + + return container_key def emit_published_datasources(self) -> Iterable[MetadataWorkUnit]: datasource_filter = f"{tableau_constant.ID_WITH_IN}: {json.dumps(self.datasource_ids_being_used)}" @@ -1695,11 +1756,22 @@ def emit_table( dataset_snapshot.aspects.append(browse_paths) else: logger.debug(f"Browse path not set for table {database_table.urn}") - schema_metadata = None + + schema_metadata = self.get_schema_metadata_for_table(columns or []) + if schema_metadata is not None: + dataset_snapshot.aspects.append(schema_metadata) + + yield self.get_metadata_change_event(dataset_snapshot) + + def get_schema_metadata_for_table( + self, columns: List[dict] + ) -> Optional[SchemaMetadata]: + schema_metadata: Optional[SchemaMetadata] = None if columns: fields = [] for field in columns: if field.get(tableau_constant.NAME) is None: + self.report.num_table_field_skipped_no_name += 1 logger.warning( f"Skipping field {field[tableau_constant.ID]} from schema since its name is none" ) @@ -1726,10 +1798,8 @@ def emit_table( hash="", platformSchema=OtherSchema(rawSchema=""), ) - if schema_metadata is not None: - dataset_snapshot.aspects.append(schema_metadata) - yield self.get_metadata_change_event(dataset_snapshot) + return schema_metadata def get_sheetwise_upstream_datasources(self, sheet: dict) -> set: sheet_upstream_datasources = set() @@ -2368,5 +2438,5 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: reason=f"Unable to retrieve metadata from tableau. Information: {str(md_exception)}", ) - def get_report(self) -> StaleEntityRemovalSourceReport: + def get_report(self) -> TableauSourceReport: return self.report diff --git a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py index 09411670edb0c..8615545ffbd57 100644 --- a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py +++ b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py @@ -188,6 +188,13 @@ def side_effect_workbook_data(*arg, **kwargs): ], mock_pagination +def side_effect_datasource_get_by_id(id, *arg, **kwargs): + datasources, _ = side_effect_datasource_data() + for ds in datasources: + if ds._id == id: + return ds + + def tableau_ingest_common( pytestconfig, tmp_path, @@ -198,6 +205,7 @@ def tableau_ingest_common( pipeline_config=config_source_default, sign_out_side_effect=lambda: None, pipeline_name="tableau-test-pipeline", + datasources_side_effect=side_effect_datasource_data, ): with mock.patch( "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph", @@ -215,7 +223,10 @@ def tableau_ingest_common( mock_client.projects = mock.Mock() mock_client.projects.get.side_effect = side_effect_project_data mock_client.datasources = mock.Mock() - mock_client.datasources.get.side_effect = side_effect_datasource_data + mock_client.datasources.get.side_effect = datasources_side_effect + mock_client.datasources.get_by_id.side_effect = ( + side_effect_datasource_get_by_id + ) mock_client.workbooks = mock.Mock() mock_client.workbooks.get.side_effect = side_effect_workbook_data mock_client.views.get.side_effect = side_effect_usage_stat @@ -243,12 +254,13 @@ def tableau_ingest_common( pipeline.run() pipeline.raise_from_status() - mce_helpers.check_golden_file( - pytestconfig, - output_path=f"{tmp_path}/{output_file_name}", - golden_path=test_resources_dir / golden_file_name, - ignore_paths=mce_helpers.IGNORE_PATH_TIMESTAMPS, - ) + if golden_file_name: + mce_helpers.check_golden_file( + pytestconfig, + output_path=f"{tmp_path}/{output_file_name}", + golden_path=test_resources_dir / golden_file_name, + ignore_paths=mce_helpers.IGNORE_PATH_TIMESTAMPS, + ) return pipeline @@ -767,3 +779,28 @@ def test_tableau_unsupported_csql(mock_datahub_graph): mcp.entityUrn == "urn:li:dataset:(urn:li:dataPlatform:tableau,09988088-05ad-173c-a2f1-f33ba3a13d1a,PROD)" ) + + +@freeze_time(FROZEN_TIME) +@pytest.mark.integration +def test_get_all_datasources_failure(pytestconfig, tmp_path, mock_datahub_graph): + output_file_name: str = "tableau_mces.json" + golden_file_name: str = "tableau_mces_golden.json" + tableau_ingest_common( + pytestconfig, + tmp_path, + [ + read_response(pytestconfig, "workbooksConnection_all.json"), + read_response(pytestconfig, "sheetsConnection_all.json"), + read_response(pytestconfig, "dashboardsConnection_all.json"), + read_response(pytestconfig, "embeddedDatasourcesConnection_all.json"), + read_response(pytestconfig, "publishedDatasourcesConnection_all.json"), + read_response(pytestconfig, "customSQLTablesConnection_all.json"), + read_response(pytestconfig, "databaseTablesConnection_all.json"), + ], + golden_file_name, + output_file_name, + mock_datahub_graph, + pipeline_name="test_tableau_ingest", + datasources_side_effect=ValueError("project_id must be defined."), + ) From e32d5520393c441f0aecbe8bba3688dcd1ac0092 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Thu, 13 Jul 2023 08:28:43 +0100 Subject: [PATCH 138/222] fix(owner): Corrects ownership aspect generation during update operations (#8399) --- .../types/common/mappers/OwnerUpdateMapper.java | 15 +++++++++++---- .../linkedin/metadata/service/OwnerService.java | 8 ++++++++ .../metadata/service/OwnerServiceTest.java | 6 +++++- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/OwnerUpdateMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/OwnerUpdateMapper.java index aac0067974a69..d978abee5bdfc 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/OwnerUpdateMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/OwnerUpdateMapper.java @@ -37,10 +37,17 @@ public Owner apply(@Nonnull final OwnerUpdate input) { } if (input.getOwnershipTypeUrn() != null) { owner.setTypeUrn(UrnUtils.getUrn(input.getOwnershipTypeUrn())); - } else if (input.getType() != null) { - owner.setType(OwnershipType.valueOf(input.getType().toString())); - } else { - throw new RuntimeException("Ownership type not specified. Please define the ownership type urn."); + } + // For backwards compatibility we have to always set the deprecated type. + // If the type exists we assume it's an old ownership type that we can map to. + // Else if it's a net new custom ownership type set old type to CUSTOM. + OwnershipType type = input.getType() != null ? OwnershipType.valueOf(input.getType().toString()) + : OwnershipType.CUSTOM; + owner.setType(type); + + if (input.getOwnershipTypeUrn() != null) { + owner.setTypeUrn(UrnUtils.getUrn(input.getOwnershipTypeUrn())); + owner.setType(OwnershipType.CUSTOM); } owner.setSource(new OwnershipSource().setType(OwnershipSourceType.SERVICE)); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/OwnerService.java b/metadata-io/src/main/java/com/linkedin/metadata/service/OwnerService.java index 6eede5c9c1f92..7385e8aa6acae 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/service/OwnerService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/service/OwnerService.java @@ -26,6 +26,8 @@ @Slf4j public class OwnerService extends BaseService { + public static final String SYSTEM_ID = "__system__"; + public OwnerService(@Nonnull EntityClient entityClient, @Nonnull Authentication systemAuthentication) { super(entityClient, systemAuthentication); } @@ -205,10 +207,16 @@ private void addOwnersIfNotExists(Ownership owners, List ownerUrns, Ownersh for (Urn ownerUrn : ownersToAdd) { Owner newOwner = new Owner(); newOwner.setOwner(ownerUrn); + newOwner.setTypeUrn(mapOwnershipTypeToEntity(OwnershipType.NONE.name())); newOwner.setType(ownershipType); ownerAssociationArray.add(newOwner); } } + @VisibleForTesting + static Urn mapOwnershipTypeToEntity(String type) { + final String typeName = SYSTEM_ID + type.toLowerCase(); + return Urn.createFromTuple(Constants.OWNERSHIP_TYPE_ENTITY_NAME, typeName); + } private static OwnerArray removeOwnersIfExists(Ownership owners, List ownerUrns) { if (!owners.hasOwners()) { diff --git a/metadata-io/src/test/java/com/linkedin/metadata/service/OwnerServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/service/OwnerServiceTest.java index 6721f59ffdd23..9247837fe0b32 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/service/OwnerServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/service/OwnerServiceTest.java @@ -29,6 +29,8 @@ import org.testng.Assert; import org.testng.annotations.Test; +import static com.linkedin.metadata.service.OwnerService.*; + public class OwnerServiceTest { @@ -63,6 +65,7 @@ private void testAddOwnersExistingOwner() throws Exception { ImmutableList.of( new Owner().setOwner(TEST_OWNER_URN_1).setType(OwnershipType.NONE), new Owner().setOwner(newOwnerUrn).setType(OwnershipType.NONE) + .setTypeUrn(mapOwnershipTypeToEntity(OwnershipType.NONE.toString())) )); MetadataChangeProposal event1 = events.get(0); @@ -102,7 +105,8 @@ private void testAddOwnersNoExistingOwners() throws Exception { mockAuthentication()); OwnerArray expectedOwners = new OwnerArray( - ImmutableList.of(new Owner().setOwner(newOwnerUrn).setType(OwnershipType.NONE))); + ImmutableList.of(new Owner().setOwner(newOwnerUrn).setType(OwnershipType.NONE) + .setTypeUrn(mapOwnershipTypeToEntity(OwnershipType.NONE.toString())))); MetadataChangeProposal event1 = events.get(0); Assert.assertEquals(event1.getAspectName(), Constants.OWNERSHIP_ASPECT_NAME); From a8d822d87416ce954ea2c8834ac3940d4d604d81 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Thu, 13 Jul 2023 13:23:47 +0530 Subject: [PATCH 139/222] chore(stats): change default stats lookback (#8408) --- .../dataset/profile/stats/historical/HistoricalStatsView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datahub-web-react/src/app/entity/dataset/profile/stats/historical/HistoricalStatsView.tsx b/datahub-web-react/src/app/entity/dataset/profile/stats/historical/HistoricalStatsView.tsx index 7a28020291a22..4b256ab5eee01 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/stats/historical/HistoricalStatsView.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/stats/historical/HistoricalStatsView.tsx @@ -104,7 +104,7 @@ const LOOKBACK_WINDOWS = [ { text: '1 year', windowSize: { interval: DateInterval.Year, count: 1 } }, ]; -const DEFAULT_LOOKBACK_WINDOW = '1 week'; +const DEFAULT_LOOKBACK_WINDOW = '3 months'; const getLookbackWindowSize = (text: string) => { for (let i = 0; i < LOOKBACK_WINDOWS.length; i++) { From a48855796256dcbca6f0a20fe18b114e2e1c6155 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Thu, 13 Jul 2023 16:00:42 +0530 Subject: [PATCH 140/222] =?UTF-8?q?feat(ingest/kafka-connect):=20allow=20s?= =?UTF-8?q?etting=20platform=5Finstance=20for=20kafka=E2=80=A6=20(#8299)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datahub/ingestion/source/kafka_connect.py | 172 +++++++++--------- .../kafka_connect_mces_golden.json | 2 +- .../kafka_connect_mongo_mces_golden.json | 2 +- .../kafka_connect_mongo_to_file.yml | 1 + .../kafka-connect/kafka_connect_to_file.yml | 1 + 5 files changed, 93 insertions(+), 85 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py b/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py index bdf1a5861dd5c..8d2bae78f671e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py +++ b/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py @@ -1,6 +1,5 @@ import logging import re -import sys from dataclasses import dataclass, field from typing import Dict, Iterable, List, Optional, Tuple @@ -42,6 +41,11 @@ logger = logging.getLogger(__name__) +KAFKA = "kafka" +SOURCE = "source" +SINK = "sink" +CONNECTOR_CLASS = "connector.class" + class ProvidedConfig(ConfigModel): provider: str @@ -82,7 +86,7 @@ class KafkaConnectSourceConfig( provided_configs: Optional[List[ProvidedConfig]] = Field( default=None, description="Provided Configurations" ) - connect_to_platform_map: Optional[dict] = Field( + connect_to_platform_map: Optional[Dict[str, Dict[str, str]]] = Field( default=None, description='Platform instance mapping when multiple instances for a platform is available. Entry for a platform should be in either `platform_instance_map` or `connect_to_platform_map`. e.g.`connect_to_platform_map: { "postgres-connector-finance-db": "postgres": "core_finance_instance" }`', ) @@ -156,12 +160,9 @@ def unquote( def get_dataset_name( database_name: Optional[str], - instance_name: Optional[str], source_table: str, ) -> str: - if database_name and instance_name: - dataset_name = instance_name + "." + database_name + "." + source_table - elif database_name: + if database_name: dataset_name = database_name + "." + source_table else: dataset_name = source_table @@ -169,29 +170,26 @@ def get_dataset_name( return dataset_name -def get_instance_name( - config: KafkaConnectSourceConfig, kafka_connector_name: str, source_platform: str +def get_platform_instance( + config: KafkaConnectSourceConfig, connector_name: str, platform: str ) -> Optional[str]: instance_name = None - if config.connect_to_platform_map: - for connector_name in config.connect_to_platform_map: - if connector_name == kafka_connector_name: - instance_name = config.connect_to_platform_map[connector_name][ - source_platform - ] - if config.platform_instance_map and config.platform_instance_map.get( - source_platform - ): - logger.error( - f"Same source platform {source_platform} configured in both platform_instance_map and connect_to_platform_map" - ) - sys.exit( - "Config Error: Same source platform configured in both platform_instance_map and connect_to_platform_map. Fix the config and re-run again." - ) - logger.info( - f"Instance name assigned is: {instance_name} for Connector Name {connector_name} and source platform {source_platform}" - ) - break + if ( + config.connect_to_platform_map + and config.connect_to_platform_map.get(connector_name) + and config.connect_to_platform_map[connector_name].get(platform) + ): + instance_name = config.connect_to_platform_map[connector_name][platform] + if config.platform_instance_map and config.platform_instance_map.get(platform): + logger.warning( + f"Same source platform {platform} configured in both platform_instance_map and connect_to_platform_map." + "Will prefer connector specific platform instance from connect_to_platform_map." + ) + elif config.platform_instance_map and config.platform_instance_map.get(platform): + instance_name = config.platform_instance_map[platform] + logger.info( + f"Instance name assigned is: {instance_name} for Connector Name {connector_name} and platform {platform}" + ) return instance_name @@ -335,7 +333,6 @@ def default_get_lineages( source_platform: str, topic_names: Optional[Iterable[str]] = None, include_source_dataset: bool = True, - instance_name: Optional[str] = None, ) -> List[KafkaConnectLineage]: lineages: List[KafkaConnectLineage] = [] if not topic_names: @@ -360,14 +357,12 @@ def default_get_lineages( self.connector_manifest.name, f"could not find schema for table {source_table}", ) - dataset_name: str = get_dataset_name( - database_name, instance_name, source_table - ) + dataset_name: str = get_dataset_name(database_name, source_table) lineage = KafkaConnectLineage( source_dataset=dataset_name if include_source_dataset else None, source_platform=source_platform, target_dataset=topic, - target_platform="kafka", + target_platform=KAFKA, ) lineages.append(lineage) return lineages @@ -427,9 +422,6 @@ def _extract_lineages(self): topic_prefix = parser.topic_prefix transforms = parser.transforms self.connector_manifest.flow_property_bag = self.connector_manifest.config - instance_name = get_instance_name( - self.config, self.connector_manifest.name, source_platform - ) # Mask/Remove properties that may reveal credentials self.connector_manifest.flow_property_bag[ @@ -452,15 +444,13 @@ def _extract_lineages(self): # Lineage source_table can be extracted by parsing query for topic in self.connector_manifest.topic_names: # default method - as per earlier implementation - dataset_name: str = get_dataset_name( - database_name, instance_name, topic - ) + dataset_name: str = get_dataset_name(database_name, topic) lineage = KafkaConnectLineage( source_dataset=None, source_platform=source_platform, target_dataset=topic, - target_platform="kafka", + target_platform=KAFKA, ) lineages.append(lineage) self.report_warning( @@ -492,7 +482,6 @@ def _extract_lineages(self): database_name=database_name, source_platform=source_platform, topic_prefix=topic_prefix, - instance_name=instance_name, ) return @@ -521,15 +510,13 @@ def _extract_lineages(self): if has_three_level_hierarchy(source_platform) and len(table) > 1: source_table = f"{table[-2]}.{table[-1]}" - dataset_name = get_dataset_name( - database_name, instance_name, source_table - ) + dataset_name = get_dataset_name(database_name, source_table) lineage = KafkaConnectLineage( source_dataset=dataset_name, source_platform=source_platform, target_dataset=topic, - target_platform="kafka", + target_platform=KAFKA, ) topic_names.remove(topic) lineages.append(lineage) @@ -569,7 +556,6 @@ def _extract_lineages(self): source_platform=source_platform, topic_prefix=topic_prefix, include_source_dataset=include_source_dataset, - instance_name=instance_name, ) self.connector_manifest.lineages = lineages return @@ -625,13 +611,13 @@ def _extract_lineages(self): found = re.search(re.compile(topic_naming_pattern), topic) if found: - table_name = get_dataset_name(found.group(1), None, found.group(2)) + table_name = get_dataset_name(found.group(1), found.group(2)) lineage = KafkaConnectLineage( source_dataset=table_name, source_platform=source_platform, target_dataset=topic, - target_platform="kafka", + target_platform=KAFKA, ) lineages.append(lineage) self.connector_manifest.lineages = lineages @@ -664,7 +650,7 @@ def get_parser( self, connector_manifest: ConnectorManifest, ) -> DebeziumParser: - connector_class = connector_manifest.config.get("connector.class", "") + connector_class = connector_manifest.config.get(CONNECTOR_CLASS, "") if connector_class == "io.debezium.connector.mysql.MySqlConnector": parser = self.DebeziumParser( @@ -726,27 +712,21 @@ def _extract_lineages(self): server_name = parser.server_name database_name = parser.database_name topic_naming_pattern = r"({0})\.(\w+\.\w+)".format(server_name) - instance_name = get_instance_name( - self.config, self.connector_manifest.name, source_platform - ) if not self.connector_manifest.topic_names: return lineages - # Get the platform/platform_instance mapping for every database_server from connect_to_platform_map for topic in self.connector_manifest.topic_names: found = re.search(re.compile(topic_naming_pattern), topic) if found: - table_name = get_dataset_name( - database_name, instance_name, found.group(2) - ) + table_name = get_dataset_name(database_name, found.group(2)) lineage = KafkaConnectLineage( source_dataset=table_name, source_platform=source_platform, target_dataset=topic, - target_platform="kafka", + target_platform=KAFKA, ) lineages.append(lineage) self.connector_manifest.lineages = lineages @@ -890,7 +870,7 @@ def _extract_lineages(self): lineages.append( KafkaConnectLineage( source_dataset=topic, - source_platform="kafka", + source_platform=KAFKA, target_dataset=target_dataset, target_platform=target_platform, ) @@ -987,7 +967,7 @@ def get_connectors_manifest(self) -> List[ConnectorManifest]: connector_manifest.topic_names = topics[c]["topics"] # Populate Source Connector metadata - if connector_manifest.type == "source": + if connector_manifest.type == SOURCE: tasks = self.session.get( f"{self.config.connect_uri}/connectors/{c}/tasks", ).json() @@ -995,7 +975,7 @@ def get_connectors_manifest(self) -> List[ConnectorManifest]: connector_manifest.tasks = tasks # JDBC source connector lineages - if connector_manifest.config.get("connector.class").__eq__( + if connector_manifest.config.get(CONNECTOR_CLASS).__eq__( "io.confluent.connect.jdbc.JdbcSourceConnector" ): connector_manifest = ConfluentJDBCSourceConnector( @@ -1003,14 +983,14 @@ def get_connectors_manifest(self) -> List[ConnectorManifest]: config=self.config, report=self.report, ).connector_manifest - elif connector_manifest.config.get("connector.class", "").startswith( + elif connector_manifest.config.get(CONNECTOR_CLASS, "").startswith( "io.debezium.connector" ): connector_manifest = DebeziumSourceConnector( connector_manifest=connector_manifest, config=self.config ).connector_manifest elif ( - connector_manifest.config.get("connector.class", "") + connector_manifest.config.get(CONNECTOR_CLASS, "") == "com.mongodb.kafka.connect.MongoSourceConnector" ): connector_manifest = MongoSourceConnector( @@ -1034,13 +1014,13 @@ def get_connectors_manifest(self) -> List[ConnectorManifest]: source_dataset=target_connector.source_dataset, source_platform=target_connector.source_platform, target_dataset=topic, - target_platform="kafka", + target_platform=KAFKA, ) connector_manifest.lineages.append(lineage) - if connector_manifest.type == "sink": - if connector_manifest.config.get("connector.class").__eq__( + if connector_manifest.type == SINK: + if connector_manifest.config.get(CONNECTOR_CLASS).__eq__( "com.wepay.kafka.connect.bigquery.BigQuerySinkConnector" ): connector_manifest = BigQuerySinkConnector( @@ -1060,7 +1040,7 @@ def get_connectors_manifest(self) -> List[ConnectorManifest]: def construct_flow_workunit(self, connector: ConnectorManifest) -> MetadataWorkUnit: connector_name = connector.name connector_type = connector.type - connector_class = connector.config.get("connector.class") + connector_class = connector.config.get(CONNECTOR_CLASS) flow_property_bag = connector.flow_property_bag # connector_url = connector.url # NOTE: this will expose connector credential when used flow_urn = builder.make_data_flow_urn( @@ -1100,18 +1080,20 @@ def construct_job_workunits( target_platform = lineage.target_platform job_property_bag = lineage.job_property_bag - job_id = ( - source_dataset - if source_dataset - else f"unknown_source.{target_dataset}" + source_platform_instance = get_platform_instance( + self.config, connector_name, source_platform ) + target_platform_instance = get_platform_instance( + self.config, connector_name, target_platform + ) + + job_id = self.get_job_id(lineage, connector, self.config) job_urn = builder.make_data_job_urn_with_flow(flow_urn, job_id) inlets = ( [ self.make_lineage_dataset_urn( - source_platform, - source_dataset, + source_platform, source_dataset, source_platform_instance ) ] if source_dataset @@ -1119,8 +1101,7 @@ def construct_job_workunits( ) outlets = [ self.make_lineage_dataset_urn( - target_platform, - target_dataset, + target_platform, target_dataset, target_platform_instance ) ] @@ -1129,9 +1110,7 @@ def construct_job_workunits( aspect=models.DataJobInfoClass( name=f"{connector_name}:{job_id}", type="COMMAND", - description=None, - customProperties=job_property_bag - # externalUrl=job_url, + customProperties=job_property_bag, ), ).as_workunit() @@ -1143,6 +1122,37 @@ def construct_job_workunits( ), ).as_workunit() + def get_job_id( + self, + lineage: KafkaConnectLineage, + connector: ConnectorManifest, + config: KafkaConnectSourceConfig, + ) -> str: + connector_class = connector.config.get(CONNECTOR_CLASS) + + # Note - This block is only to maintain backward compatibility of Job URN + if ( + connector_class + and connector.type == SOURCE + and ( + "JdbcSourceConnector" in connector_class + or connector_class.startswith("io.debezium.connector") + ) + and lineage.source_dataset + and config.connect_to_platform_map + and config.connect_to_platform_map.get(connector.name) + and config.connect_to_platform_map[connector.name].get( + lineage.source_platform + ) + ): + return f"{config.connect_to_platform_map[connector.name][lineage.source_platform]}.{lineage.source_dataset}" + + return ( + lineage.source_dataset + if lineage.source_dataset + else f"unknown_source.{lineage.target_dataset}" + ) + def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: return [ *super().get_workunit_processors(), @@ -1163,16 +1173,12 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: def get_report(self) -> KafkaConnectSourceReport: return self.report - def make_lineage_dataset_urn(self, platform: str, name: str) -> str: + def make_lineage_dataset_urn( + self, platform: str, name: str, platform_instance: Optional[str] + ) -> str: if self.config.convert_lineage_urns_to_lowercase: name = name.lower() - platform_instance = ( - self.config.platform_instance_map.get(platform) - if self.config.platform_instance_map - else None - ) - return builder.make_dataset_urn_with_platform_instance( platform, name, platform_instance, self.config.env ) diff --git a/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_mces_golden.json b/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_mces_golden.json index 15d528a1873ea..1c7f481e7063e 100644 --- a/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_mces_golden.json +++ b/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_mces_golden.json @@ -206,7 +206,7 @@ "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.member,PROD)" ], "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:kafka,test-postgres-jdbc-member,PROD)" + "urn:li:dataset:(urn:li:dataPlatform:kafka,kafka1.test-postgres-jdbc-member,PROD)" ] } }, diff --git a/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_mongo_mces_golden.json b/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_mongo_mces_golden.json index 7ecda38299cc5..b947d71c1b87d 100644 --- a/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_mongo_mces_golden.json +++ b/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_mongo_mces_golden.json @@ -46,7 +46,7 @@ "urn:li:dataset:(urn:li:dataPlatform:mongodb,test_db.purchases,PROD)" ], "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:kafka,mongodb.test_db.purchases,PROD)" + "urn:li:dataset:(urn:li:dataPlatform:kafka,kafkax.mongodb.test_db.purchases,PROD)" ] } }, diff --git a/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_mongo_to_file.yml b/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_mongo_to_file.yml index 2450903cee56b..99e98952c2736 100644 --- a/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_mongo_to_file.yml +++ b/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_mongo_to_file.yml @@ -19,6 +19,7 @@ source: convert_lineage_urns_to_lowercase: true platform_instance_map: # optional mysql: mysql1 # optional + kafka: kafkax connect_to_platform_map: # optional postgres_source: # optional - Connector name postgres: postgres1 # optional - Platform to instance map diff --git a/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_to_file.yml b/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_to_file.yml index 5a959a2f63378..67ec384efcbf8 100644 --- a/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_to_file.yml +++ b/metadata-ingestion/tests/integration/kafka-connect/kafka_connect_to_file.yml @@ -22,6 +22,7 @@ source: connect_to_platform_map: # optional postgres_source: # optional - Connector name postgres: postgres1 # optional - Platform to instance map + kafka: kafka1 generic_connectors: # optional - Generic connector lineage info - connector_name: generic-connector source_dataset: generic-dataset From cbbe0837319edfa908d9135a15e4cfdfd9a8f5a8 Mon Sep 17 00:00:00 2001 From: mohdsiddique Date: Thu, 13 Jul 2023 17:33:19 +0530 Subject: [PATCH 141/222] fix(ingestion/powerbi): increment msal version (#8385) Co-authored-by: MohdSiddiqueBagwan --- metadata-ingestion/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 93925b3ec96f1..9aca187a8d6b6 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -212,7 +212,7 @@ def get_long_description(): "trino[sqlalchemy]>=0.308, !=0.317", } -microsoft_common = {"msal==1.16.0"} +microsoft_common = {"msal==1.22.0"} iceberg_common = { # Iceberg Python SDK From 83ebeb207debf16f5064b40c08856a184d5c6842 Mon Sep 17 00:00:00 2001 From: Ellie O'Neil <110510035+eboneil@users.noreply.github.com> Date: Thu, 13 Jul 2023 05:04:26 -0700 Subject: [PATCH 142/222] docs(perf-test) Update README (#8410) --- perf-test/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/perf-test/README.md b/perf-test/README.md index 81fdad2355741..24fb064d3e28a 100644 --- a/perf-test/README.md +++ b/perf-test/README.md @@ -17,6 +17,11 @@ pip3 install locust Note that it supports python versions 3.6 and up. Refer to this [guide](https://docs.locust.io/en/stable/installation.html) for more details. +You will also need to import requirements in order to run the Locustfile scripts: +```shell +pip3 install -r requirements.txt +``` + ## Locustfiles [Locustfiles](./locustfiles) define how the users will behave once they are spawned. Refer to From a91c78cf314fe0abe7acbe68960fffaf66a8bf1c Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Fri, 14 Jul 2023 00:42:00 +0200 Subject: [PATCH 143/222] fix(ingest/s3): fix test flakiness (#8416) --- .../s3/golden_mces_file_without_extension.json | 2 +- .../s3/golden_mces_folder_no_partition.json | 8 ++++---- .../golden_mces_folder_no_partition_exclude.json | 4 ++-- .../golden_mces_folder_no_partition_filename.json | 8 ++++---- .../s3/golden_mces_folder_no_partition_glob.json | 4 ++-- .../s3/golden_mces_folder_partition_basic.json | 2 +- .../s3/golden_mces_folder_partition_keyval.json | 2 +- ...olden_mces_folder_partition_update_schema.json | 2 +- .../s3/golden_mces_multiple_files.json | 14 +++++++------- .../s3/golden_mces_multiple_spec_for_files.json | 4 ++-- ..._mces_multiple_specs_of_different_buckets.json | 4 ++-- .../golden-files/s3/golden_mces_single_file.json | 2 +- .../tests/integration/s3/test_s3.py | 15 ++++++++++++++- 13 files changed, 42 insertions(+), 29 deletions(-) diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json index 5518e93600843..cb7ccf87f7c6f 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json @@ -127,7 +127,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419674000 + "lastUpdatedTimestamp": 1586848010000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json index 1b2ef998bd5ba..0eefc4cafbd62 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json @@ -7,7 +7,7 @@ "aspect": { "json": { "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv/part1.csv" + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv/part2.csv" }, "name": "food_csv", "description": "", @@ -125,7 +125,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419673000 + "lastUpdatedTimestamp": 1586847990000 } }, "systemMetadata": { @@ -588,7 +588,7 @@ "aspect": { "json": { "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part1.parquet" + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part2.parquet" }, "name": "food_parquet", "description": "", @@ -706,7 +706,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419671000 + "lastUpdatedTimestamp": 1586847810000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json index fe50a19546653..36f40cc918735 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json @@ -7,7 +7,7 @@ "aspect": { "json": { "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part1.parquet" + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part2.parquet" }, "name": "food_parquet", "description": "", @@ -125,7 +125,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419671000 + "lastUpdatedTimestamp": 1586847810000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json index 6a1b339b2e188..7a59a7f21f118 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json @@ -7,7 +7,7 @@ "aspect": { "json": { "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv/part1.csv" + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv/part2.csv" }, "name": "folder_aaa.food_csv", "description": "", @@ -125,7 +125,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419673000 + "lastUpdatedTimestamp": 1586847990000 } }, "systemMetadata": { @@ -588,7 +588,7 @@ "aspect": { "json": { "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part1.parquet" + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part2.parquet" }, "name": "folder_aaa.food_parquet", "description": "", @@ -706,7 +706,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419671000 + "lastUpdatedTimestamp": 1586847810000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json index 071ef3efffcea..0dc71b900b03b 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json @@ -7,7 +7,7 @@ "aspect": { "json": { "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part1.parquet" + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part2.parquet" }, "name": "food_parquet", "description": "", @@ -125,7 +125,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419671000 + "lastUpdatedTimestamp": 1586847810000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json index 188369c92db46..aa683a2bcd4c2 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json @@ -557,7 +557,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419673000 + "lastUpdatedTimestamp": 1586847960000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json index 5ab5f043e9621..095c1cbe3152b 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json @@ -557,7 +557,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419673000 + "lastUpdatedTimestamp": 1586847960000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json index 6e42b68f1fa95..a5f135c30369a 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json @@ -557,7 +557,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419673000 + "lastUpdatedTimestamp": 1586847960000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json index 01ff0030e33c9..40e45150fa211 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json @@ -439,7 +439,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419669000 + "lastUpdatedTimestamp": 1586847610000 } }, "systemMetadata": { @@ -1030,7 +1030,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419668000 + "lastUpdatedTimestamp": 1586847620000 } }, "systemMetadata": { @@ -1213,7 +1213,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419668000 + "lastUpdatedTimestamp": 1586847630000 } }, "systemMetadata": { @@ -1372,7 +1372,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419668000 + "lastUpdatedTimestamp": 1586847640000 } }, "systemMetadata": { @@ -1555,7 +1555,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419668000 + "lastUpdatedTimestamp": 1586847650000 } }, "systemMetadata": { @@ -1738,7 +1738,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419668000 + "lastUpdatedTimestamp": 1586847660000 } }, "systemMetadata": { @@ -2233,7 +2233,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419668000 + "lastUpdatedTimestamp": 1586847670000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_spec_for_files.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_spec_for_files.json index 26d1636eae852..84ace7d673676 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_spec_for_files.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_spec_for_files.json @@ -149,7 +149,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419668000 + "lastUpdatedTimestamp": 1586847620000 } }, "systemMetadata": { @@ -754,7 +754,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419668000 + "lastUpdatedTimestamp": 1586847630000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json index d7a831e9490ba..f7f3cb8fb743e 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json @@ -149,7 +149,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419668000 + "lastUpdatedTimestamp": 1586847620000 } }, "systemMetadata": { @@ -754,7 +754,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419674000 + "lastUpdatedTimestamp": 1586847630000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json index caf989fa376cf..5353d95ada8f7 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json @@ -149,7 +149,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419668000 + "lastUpdatedTimestamp": 1586847620000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/test_s3.py b/metadata-ingestion/tests/integration/s3/test_s3.py index 6202fa5509beb..98ae2eaa393ab 100644 --- a/metadata-ingestion/tests/integration/s3/test_s3.py +++ b/metadata-ingestion/tests/integration/s3/test_s3.py @@ -3,6 +3,7 @@ import os from datetime import datetime +import moto.s3 import pytest from boto3.session import Session from moto import mock_s3 @@ -55,6 +56,10 @@ def s3_populate(pytestconfig, s3_resource, s3_client, bucket_names): test_resources_dir = ( pytestconfig.rootpath / "tests/integration/s3/test_data/local_system/" ) + + current_time_sec = datetime.strptime( + FROZEN_TIME, "%Y-%m-%d %H:%M:%S" + ).timestamp() for root, _dirs, files in os.walk(test_resources_dir): for file in sorted(files): full_path = os.path.join(root, file) @@ -65,6 +70,13 @@ def s3_populate(pytestconfig, s3_resource, s3_client, bucket_names): Key=rel_path, Tagging={"TagSet": [{"Key": "baz", "Value": "bob"}]}, ) + key = ( + moto.s3.models.s3_backends["123456789012"]["global"] + .buckets[bucket_name] + .keys[rel_path] + ) + current_time_sec += 10 + key.last_modified = datetime.fromtimestamp(current_time_sec) yield @@ -73,7 +85,8 @@ def touch_local_files(pytestconfig): test_resources_dir = ( pytestconfig.rootpath / "tests/integration/s3/test_data/local_system/" ) - current_time_sec = datetime.now().timestamp() + current_time_sec = datetime.strptime(FROZEN_TIME, "%Y-%m-%d %H:%M:%S").timestamp() + for root, _dirs, files in os.walk(test_resources_dir): _dirs.sort() for file in sorted(files): From 4fb77e4a25890c84b165a1b21265c0d9dee33be5 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Fri, 14 Jul 2023 15:47:16 -0700 Subject: [PATCH 144/222] fix(ingest): tweak ingestion exit codes (#8418) --- .../src/datahub/cli/ingest_cli.py | 76 +++++++++---------- .../src/datahub/utilities/logging_manager.py | 12 +++ metadata-ingestion/tests/conftest.py | 2 + .../tests/unit/test_cli_logging.py | 6 +- .../tests/unit/test_pipeline.py | 50 ++++++++++++ 5 files changed, 107 insertions(+), 39 deletions(-) diff --git a/metadata-ingestion/src/datahub/cli/ingest_cli.py b/metadata-ingestion/src/datahub/cli/ingest_cli.py index 38f1dddeab601..c8c352d1f83ff 100644 --- a/metadata-ingestion/src/datahub/cli/ingest_cli.py +++ b/metadata-ingestion/src/datahub/cli/ingest_cli.py @@ -1,6 +1,5 @@ import asyncio import csv -import functools import json import logging import os @@ -123,9 +122,7 @@ def run( ) -> None: """Ingest metadata into DataHub.""" - def run_pipeline_to_completion( - pipeline: Pipeline, structured_report: Optional[str] = None - ) -> int: + async def run_pipeline_to_completion(pipeline: Pipeline) -> int: logger.info("Starting metadata ingestion") with click_spinner.spinner(disable=no_spinner): try: @@ -144,20 +141,44 @@ def run_pipeline_to_completion( ret = pipeline.pretty_print_summary(warnings_as_failure=strict_warnings) return ret - async def run_pipeline_async(pipeline: Pipeline) -> int: - loop = asyncio._get_running_loop() - return await loop.run_in_executor( - None, functools.partial(run_pipeline_to_completion, pipeline) + # main function begins + logger.info("DataHub CLI version: %s", datahub_package.nice_version_name()) + + pipeline_config = load_config_file( + config, + squirrel_original_config=True, + squirrel_field="__raw_config", + allow_stdin=True, + ) + raw_pipeline_config = pipeline_config.pop("__raw_config") + + if test_source_connection: + _test_source_connection(report_to, pipeline_config) + + async def run_ingestion_and_check_upgrade() -> int: + # TRICKY: We want to make sure that the Pipeline.create() call happens on the + # same thread as the rest of the ingestion. As such, we must initialize the + # pipeline inside the async function so that it happens on the same event + # loop, and hence the same thread. + + # logger.debug(f"Using config: {pipeline_config}") + pipeline = Pipeline.create( + pipeline_config, + dry_run, + preview, + preview_workunits, + report_to, + no_default_report, + raw_pipeline_config, ) - async def run_func_check_upgrade(pipeline: Pipeline) -> None: version_stats_future = asyncio.ensure_future( upgrade.retrieve_version_stats(pipeline.ctx.graph) ) - the_one_future = asyncio.ensure_future(run_pipeline_async(pipeline)) - ret = await the_one_future + ingestion_future = asyncio.ensure_future(run_pipeline_to_completion(pipeline)) + ret = await ingestion_future - # the one future has returned + # The main ingestion has completed. If it was successful, potentially show an upgrade nudge message. if ret == 0: try: # we check the other futures quickly on success @@ -167,35 +188,14 @@ async def run_func_check_upgrade(pipeline: Pipeline) -> None: logger.debug( f"timed out with {e} waiting for version stats to be computed... skipping ahead." ) - sys.exit(ret) - # main function begins - logger.info("DataHub CLI version: %s", datahub_package.nice_version_name()) - - pipeline_config = load_config_file( - config, - squirrel_original_config=True, - squirrel_field="__raw_config", - allow_stdin=True, - ) - raw_pipeline_config = pipeline_config.pop("__raw_config") - - if test_source_connection: - _test_source_connection(report_to, pipeline_config) - - # logger.debug(f"Using config: {pipeline_config}") - pipeline = Pipeline.create( - pipeline_config, - dry_run, - preview, - preview_workunits, - report_to, - no_default_report, - raw_pipeline_config, - ) + return ret loop = asyncio.get_event_loop() - loop.run_until_complete(run_func_check_upgrade(pipeline)) + ret = loop.run_until_complete(run_ingestion_and_check_upgrade()) + if ret: + sys.exit(ret) + # don't raise SystemExit if there's no error def _test_source_connection(report_to: Optional[str], pipeline_config: dict) -> None: diff --git a/metadata-ingestion/src/datahub/utilities/logging_manager.py b/metadata-ingestion/src/datahub/utilities/logging_manager.py index 8b8fd076687a5..a8eacb0a9938d 100644 --- a/metadata-ingestion/src/datahub/utilities/logging_manager.py +++ b/metadata-ingestion/src/datahub/utilities/logging_manager.py @@ -139,6 +139,11 @@ def get_log_buffer() -> _LogBuffer: def configure_logging(debug: bool, log_file: Optional[str] = None) -> Iterator[None]: _log_buffer.clear() + if os.environ.get("DATAHUB_SUPPRESS_LOGGING_MANAGER") == "1": + # If we're running in pytest, we don't want to configure logging. + yield + return + with contextlib.ExitStack() as stack: # Create stdout handler. stream_handler = logging.StreamHandler() @@ -190,6 +195,13 @@ def configure_logging(debug: bool, log_file: Optional[str] = None) -> Iterator[N yield + # Cleanup. + for handler in handlers: + root_logger.removeHandler(handler) + for lib in DATAHUB_PACKAGES: + lib_logger.removeHandler(handler) + lib_logger.propagate = True + # Reduce logging from some particularly chatty libraries. logging.getLogger("urllib3").setLevel(logging.ERROR) diff --git a/metadata-ingestion/tests/conftest.py b/metadata-ingestion/tests/conftest.py index c151987c13cc1..0eb9ab250339c 100644 --- a/metadata-ingestion/tests/conftest.py +++ b/metadata-ingestion/tests/conftest.py @@ -4,6 +4,8 @@ import pytest +os.environ["DATAHUB_SUPPRESS_LOGGING_MANAGER"] = "1" + # Enable debug logging. logging.getLogger().setLevel(logging.DEBUG) os.environ["DATAHUB_DEBUG"] = "1" diff --git a/metadata-ingestion/tests/unit/test_cli_logging.py b/metadata-ingestion/tests/unit/test_cli_logging.py index 761b72e5e0c1b..5acb5f3e3608b 100644 --- a/metadata-ingestion/tests/unit/test_cli_logging.py +++ b/metadata-ingestion/tests/unit/test_cli_logging.py @@ -9,9 +9,13 @@ from datahub.entrypoints import datahub from datahub.utilities.logging_manager import DATAHUB_PACKAGES, get_log_buffer +pytestmark = pytest.mark.skip(reason="Reconfiguring logging messes up pytest") + @pytest.fixture(autouse=True, scope="module") -def cleanup(): +def cleanup(monkeypatch): + monkeypatch.setenv("DATAHUB_SUPPRESS_LOGGING_MANAGER", "0") + """Attempt to clear undo the stateful changes done by invoking `my_logging_fn`, which calls `configure_logging`.""" yield for lib in DATAHUB_PACKAGES: diff --git a/metadata-ingestion/tests/unit/test_pipeline.py b/metadata-ingestion/tests/unit/test_pipeline.py index dd0117c302dae..7ce78f0ab3e13 100644 --- a/metadata-ingestion/tests/unit/test_pipeline.py +++ b/metadata-ingestion/tests/unit/test_pipeline.py @@ -1,3 +1,4 @@ +import pathlib from typing import Any, Iterable, List, Optional, cast from unittest.mock import patch @@ -18,6 +19,7 @@ MetadataChangeEventClass, StatusClass, ) +from tests.test_helpers.click_helpers import run_datahub_cmd from tests.test_helpers.sink_helpers import RecordingSinkReport FROZEN_TIME = "2020-04-14 07:00:00" @@ -220,6 +222,54 @@ def test_run_including_registered_transformation(self): ) assert pipeline + @pytest.mark.parametrize( + "source,strict_warnings,exit_code", + [ + pytest.param( + "FakeSource", + False, + 0, + ), + pytest.param( + "FakeSourceWithWarnings", + False, + 0, + ), + pytest.param( + "FakeSourceWithWarnings", + True, + 1, + ), + ], + ) + @freeze_time(FROZEN_TIME) + def test_pipeline_return_code(self, tmp_path, source, strict_warnings, exit_code): + config_file: pathlib.Path = tmp_path / "test.yml" + + config_file.write_text( + f""" +--- +run_id: pipeline_test +source: + type: tests.unit.test_pipeline.{source} + config: {{}} +sink: + type: console +""" + ) + + res = run_datahub_cmd( + [ + "ingest", + "-c", + f"{config_file}", + *(("--strict-warnings",) if strict_warnings else ()), + ], + tmp_path=tmp_path, + check_result=False, + ) + assert res.exit_code == exit_code, res.stdout + @pytest.mark.parametrize( "commit_policy,source,should_commit", [ From f41f642eaf4ff602c8299426912a63d26dc91e35 Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Fri, 14 Jul 2023 18:48:04 -0400 Subject: [PATCH 145/222] build(ingest/boto3): Update boto3-stubs to fix CI (#8425) --- metadata-ingestion/setup.py | 2 +- .../sagemaker_processors/feature_groups.py | 16 ++++----- .../aws/sagemaker_processors/lineage.py | 22 +++++++----- .../source/aws/sagemaker_processors/models.py | 34 +++++++++---------- 4 files changed, 39 insertions(+), 35 deletions(-) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 9aca187a8d6b6..e8f233ca15ab3 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -413,7 +413,7 @@ def get_long_description(): "types-cachetools", # versions 0.1.13 and 0.1.14 seem to have issues "types-click==0.1.12", - "boto3-stubs[s3,glue,sagemaker,sts]", + "boto3-stubs[s3,glue,sagemaker,sts]>=1.28.3", "types-tabulate", # avrogen package requires this "types-pytz", diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py index b8b96c6306a3b..207b73d4cbd46 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py @@ -23,9 +23,9 @@ if TYPE_CHECKING: from mypy_boto3_sagemaker import SageMakerClient from mypy_boto3_sagemaker.type_defs import ( - DescribeFeatureGroupResponseTypeDef, - FeatureDefinitionTypeDef, - FeatureGroupSummaryTypeDef, + DescribeFeatureGroupResponseOutputTypeDef, + FeatureDefinitionOutputTypeDef, + FeatureGroupSummaryOutputTypeDef, ) @@ -35,7 +35,7 @@ class FeatureGroupProcessor: env: str report: SagemakerSourceReport - def get_all_feature_groups(self) -> List["FeatureGroupSummaryTypeDef"]: + def get_all_feature_groups(self) -> List["FeatureGroupSummaryOutputTypeDef"]: """ List all feature groups in SageMaker. """ @@ -51,7 +51,7 @@ def get_all_feature_groups(self) -> List["FeatureGroupSummaryTypeDef"]: def get_feature_group_details( self, feature_group_name: str - ) -> "DescribeFeatureGroupResponseTypeDef": + ) -> "DescribeFeatureGroupResponseOutputTypeDef": """ Get details of a feature group (including list of component features). """ @@ -75,7 +75,7 @@ def get_feature_group_details( return feature_group def get_feature_group_wu( - self, feature_group_details: "DescribeFeatureGroupResponseTypeDef" + self, feature_group_details: "DescribeFeatureGroupResponseOutputTypeDef" ) -> MetadataWorkUnit: """ Generate an MLFeatureTable workunit for a SageMaker feature group. @@ -146,8 +146,8 @@ def get_feature_type(self, aws_type: str, feature_name: str) -> str: def get_feature_wu( self, - feature_group_details: "DescribeFeatureGroupResponseTypeDef", - feature: "FeatureDefinitionTypeDef", + feature_group_details: "DescribeFeatureGroupResponseOutputTypeDef", + feature: "FeatureDefinitionOutputTypeDef", ) -> MetadataWorkUnit: """ Generate an MLFeature workunit for a SageMaker feature. diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/lineage.py index b677dccad24ac..7ba615d80346f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/lineage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/lineage.py @@ -9,10 +9,10 @@ if TYPE_CHECKING: from mypy_boto3_sagemaker import SageMakerClient from mypy_boto3_sagemaker.type_defs import ( - ActionSummaryTypeDef, - ArtifactSummaryTypeDef, - AssociationSummaryTypeDef, - ContextSummaryTypeDef, + ActionSummaryOutputTypeDef, + ArtifactSummaryOutputTypeDef, + AssociationSummaryOutputTypeDef, + ContextSummaryOutputTypeDef, ) @@ -49,7 +49,7 @@ class LineageProcessor: nodes: Dict[str, Dict[str, Any]] = field(default_factory=dict) lineage_info: LineageInfo = field(default_factory=LineageInfo) - def get_all_actions(self) -> List["ActionSummaryTypeDef"]: + def get_all_actions(self) -> List["ActionSummaryOutputTypeDef"]: """ List all actions in SageMaker. """ @@ -63,7 +63,7 @@ def get_all_actions(self) -> List["ActionSummaryTypeDef"]: return actions - def get_all_artifacts(self) -> List["ArtifactSummaryTypeDef"]: + def get_all_artifacts(self) -> List["ArtifactSummaryOutputTypeDef"]: """ List all artifacts in SageMaker. """ @@ -77,7 +77,7 @@ def get_all_artifacts(self) -> List["ArtifactSummaryTypeDef"]: return artifacts - def get_all_contexts(self) -> List["ContextSummaryTypeDef"]: + def get_all_contexts(self) -> List["ContextSummaryOutputTypeDef"]: """ List all contexts in SageMaker. """ @@ -91,7 +91,9 @@ def get_all_contexts(self) -> List["ContextSummaryTypeDef"]: return contexts - def get_incoming_edges(self, node_arn: str) -> List["AssociationSummaryTypeDef"]: + def get_incoming_edges( + self, node_arn: str + ) -> List["AssociationSummaryOutputTypeDef"]: """ Get all incoming edges for a node in the lineage graph. """ @@ -105,7 +107,9 @@ def get_incoming_edges(self, node_arn: str) -> List["AssociationSummaryTypeDef"] return edges - def get_outgoing_edges(self, node_arn: str) -> List["AssociationSummaryTypeDef"]: + def get_outgoing_edges( + self, node_arn: str + ) -> List["AssociationSummaryOutputTypeDef"]: """ Get all outgoing edges for a node in the lineage graph. """ diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/models.py b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/models.py index e82cfc58f75a7..12770b0831693 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/models.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/models.py @@ -45,12 +45,12 @@ if TYPE_CHECKING: from mypy_boto3_sagemaker import SageMakerClient from mypy_boto3_sagemaker.type_defs import ( - DescribeEndpointOutputTypeDef, - DescribeModelOutputTypeDef, - DescribeModelPackageGroupOutputTypeDef, - EndpointSummaryTypeDef, - ModelPackageGroupSummaryTypeDef, - ModelSummaryTypeDef, + DescribeEndpointOutputOutputTypeDef, + DescribeModelOutputOutputTypeDef, + DescribeModelPackageGroupOutputOutputTypeDef, + EndpointSummaryOutputTypeDef, + ModelPackageGroupSummaryOutputTypeDef, + ModelSummaryOutputTypeDef, ) ENDPOINT_STATUS_MAP: Dict[str, str] = { @@ -91,7 +91,7 @@ class ModelProcessor: group_arn_to_name: Dict[str, str] = field(default_factory=dict) - def get_all_models(self) -> List["ModelSummaryTypeDef"]: + def get_all_models(self) -> List["ModelSummaryOutputTypeDef"]: """ List all models in SageMaker. """ @@ -105,7 +105,7 @@ def get_all_models(self) -> List["ModelSummaryTypeDef"]: return models - def get_model_details(self, model_name: str) -> "DescribeModelOutputTypeDef": + def get_model_details(self, model_name: str) -> "DescribeModelOutputOutputTypeDef": """ Get details of a model. """ @@ -113,7 +113,7 @@ def get_model_details(self, model_name: str) -> "DescribeModelOutputTypeDef": # see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sagemaker.html#SageMaker.Client.describe_model return self.sagemaker_client.describe_model(ModelName=model_name) - def get_all_groups(self) -> List["ModelPackageGroupSummaryTypeDef"]: + def get_all_groups(self) -> List["ModelPackageGroupSummaryOutputTypeDef"]: """ List all model groups in SageMaker. """ @@ -128,7 +128,7 @@ def get_all_groups(self) -> List["ModelPackageGroupSummaryTypeDef"]: def get_group_details( self, group_name: str - ) -> "DescribeModelPackageGroupOutputTypeDef": + ) -> "DescribeModelPackageGroupOutputOutputTypeDef": """ Get details of a model group. """ @@ -138,7 +138,7 @@ def get_group_details( ModelPackageGroupName=group_name ) - def get_all_endpoints(self) -> List["EndpointSummaryTypeDef"]: + def get_all_endpoints(self) -> List["EndpointSummaryOutputTypeDef"]: endpoints = [] # see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sagemaker.html#SageMaker.Client.list_endpoints @@ -151,7 +151,7 @@ def get_all_endpoints(self) -> List["EndpointSummaryTypeDef"]: def get_endpoint_details( self, endpoint_name: str - ) -> "DescribeEndpointOutputTypeDef": + ) -> "DescribeEndpointOutputOutputTypeDef": # see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sagemaker.html#SageMaker.Client.describe_endpoint return self.sagemaker_client.describe_endpoint(EndpointName=endpoint_name) @@ -171,7 +171,7 @@ def get_endpoint_status( return endpoint_status def get_endpoint_wu( - self, endpoint_details: "DescribeEndpointOutputTypeDef" + self, endpoint_details: "DescribeEndpointOutputOutputTypeDef" ) -> MetadataWorkUnit: """a Get a workunit for an endpoint. @@ -215,7 +215,7 @@ def get_endpoint_wu( def get_model_endpoints( self, - model_details: "DescribeModelOutputTypeDef", + model_details: "DescribeModelOutputOutputTypeDef", endpoint_arn_to_name: Dict[str, str], model_image: Optional[str], model_uri: Optional[str], @@ -244,7 +244,7 @@ def get_model_endpoints( return model_endpoints_sorted def get_group_wu( - self, group_details: "DescribeModelPackageGroupOutputTypeDef" + self, group_details: "DescribeModelPackageGroupOutputOutputTypeDef" ) -> MetadataWorkUnit: """ Get a workunit for a model group. @@ -294,7 +294,7 @@ def get_group_wu( return MetadataWorkUnit(id=group_name, mce=mce) def match_model_jobs( - self, model_details: "DescribeModelOutputTypeDef" + self, model_details: "DescribeModelOutputOutputTypeDef" ) -> Tuple[Set[str], Set[str], List[MLHyperParamClass], List[MLMetricClass]]: model_training_jobs: Set[str] = set() model_downstream_jobs: Set[str] = set() @@ -387,7 +387,7 @@ def strip_quotes(string: str) -> str: def get_model_wu( self, - model_details: "DescribeModelOutputTypeDef", + model_details: "DescribeModelOutputOutputTypeDef", endpoint_arn_to_name: Dict[str, str], ) -> MetadataWorkUnit: """ From 61611b7202359eaccbbb695d3cfb34ac17dd19f9 Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Fri, 14 Jul 2023 18:50:32 -0400 Subject: [PATCH 146/222] feat(ingest/snowflake): View CLL from sql parsing of view definition (#8419) --- .../source/snowflake/snowflake_config.py | 17 +- .../snowflake/snowflake_lineage_legacy.py | 5 +- .../source/snowflake/snowflake_lineage_v2.py | 219 +++++++++++++----- .../source/snowflake/snowflake_report.py | 5 + .../source/snowflake/snowflake_schema.py | 3 + .../source/snowflake/snowflake_v2.py | 65 ++++-- .../ingestion/source_config/sql/snowflake.py | 2 +- .../src/datahub/utilities/sqlglot_lineage.py | 6 +- 8 files changed, 232 insertions(+), 90 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py index eb7ce49bf1343..79bf538af91d2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py @@ -65,7 +65,12 @@ class SnowflakeV2Config( include_column_lineage: bool = Field( default=True, - description="If enabled, populates the column lineage. Supported only for snowflake table-to-table and view-to-table lineage edge (not supported in table-to-view or view-to-view lineage edge yet). Requires appropriate grants given to the role.", + description="Populates table->table and view->table column lineage. Requires appropriate grants given to the role and the Snowflake Enterprise Edition or above.", + ) + + include_view_column_lineage: bool = Field( + default=False, + description="Populates view->view and table->view column lineage.", ) _check_role_grants_removed = pydantic_removed_field("check_role_grants") @@ -88,7 +93,11 @@ class SnowflakeV2Config( use_legacy_lineage_method: bool = Field( default=False, - description="Whether to use the legacy lineage computation method. If set to False, ingestion uses new optimised lineage extraction method that requires less ingestion process memory.", + description=( + "Whether to use the legacy lineage computation method. " + "By default, uses new optimised lineage extraction method that requires less ingestion process memory. " + "Table-to-view and view-to-view column-level lineage are not supported with the legacy method." + ), ) validate_upstreams_against_patterns: bool = Field( @@ -179,3 +188,7 @@ def get_sql_alchemy_url( return BaseSnowflakeConfig.get_sql_alchemy_url( self, database=database, username=username, password=password, role=role ) + + @property + def parse_view_ddl(self) -> bool: + return self.include_view_column_lineage diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_legacy.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_legacy.py index 5eb05e42ef389..832a072c619f8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_legacy.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_legacy.py @@ -2,7 +2,7 @@ import logging from collections import defaultdict from dataclasses import dataclass, field -from typing import Callable, Dict, FrozenSet, Iterable, List, Optional, Set +from typing import Any, Callable, Dict, FrozenSet, Iterable, List, Optional, Set from pydantic import Field from pydantic.error_wrappers import ValidationError @@ -193,8 +193,9 @@ def __init__( self.dataset_urn_builder = dataset_urn_builder self.connection: Optional[SnowflakeConnection] = None + # Kwargs used by new snowflake lineage extractor need to be ignored here def get_workunits( - self, discovered_tables: List[str], discovered_views: List[str] + self, discovered_tables: List[str], discovered_views: List[str], **_kwargs: Any ) -> Iterable[MetadataWorkUnit]: self.connection = self.create_connection() if self.connection is None: diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py index 21e615d2cce7f..c338c427aefbf 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py @@ -1,8 +1,19 @@ import json import logging from collections import defaultdict -from dataclasses import dataclass, field -from typing import Callable, Dict, FrozenSet, Iterable, List, Optional, Set +from dataclasses import dataclass +from typing import ( + Callable, + Collection, + Dict, + Iterable, + List, + MutableMapping, + Optional, + Sequence, + Set, + Tuple, +) from snowflake.connector import SnowflakeConnection @@ -13,6 +24,7 @@ from datahub.ingestion.source.snowflake.constants import ( LINEAGE_PERMISSION_ERROR, SnowflakeEdition, + SnowflakeObjectDomain, ) from datahub.ingestion.source.snowflake.snowflake_config import SnowflakeV2Config from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery @@ -31,6 +43,12 @@ ) from datahub.metadata.schema_classes import DatasetLineageTypeClass, UpstreamClass from datahub.utilities.perf_timer import PerfTimer +from datahub.utilities.sqlglot_lineage import ( + SchemaResolver, + SqlParsingResult, + sqlglot_lineage, +) +from datahub.utilities.urns.dataset_urn import DatasetUrn logger: logging.Logger = logging.getLogger(__name__) @@ -42,25 +60,6 @@ class SnowflakeColumnId: objectDomain: Optional[str] = None -@dataclass(frozen=True) -class SnowflakeColumnFineGrainedLineage: - """ - Fie grained upstream of column, - which represents a transformation applied on input columns""" - - inputColumns: FrozenSet[SnowflakeColumnId] - # Transform function, query etc can be added here - - -@dataclass -class SnowflakeColumnUpstreams: - """All upstreams of a column""" - - upstreams: Set[SnowflakeColumnFineGrainedLineage] = field( - default_factory=set, init=False - ) - - class SnowflakeLineageExtractor( SnowflakeQueryMixin, SnowflakeConnectionMixin, SnowflakeCommonMixin ): @@ -91,7 +90,11 @@ def __init__( self.connection: Optional[SnowflakeConnection] = None def get_workunits( - self, discovered_tables: List[str], discovered_views: List[str] + self, + discovered_tables: List[str], + discovered_views: List[str], + schema_resolver: SchemaResolver, + view_definitions: MutableMapping[str, str], ) -> Iterable[MetadataWorkUnit]: self.connection = self.create_connection() if self.connection is None: @@ -101,18 +104,20 @@ def get_workunits( if self.config.include_view_lineage: if len(discovered_views) > 0: - yield from self.get_view_upstream_workunits(discovered_views) + yield from self.get_view_upstream_workunits( + discovered_views=discovered_views, + schema_resolver=schema_resolver, + view_definitions=view_definitions, + ) else: logger.info("No views found. Skipping View Lineage Extraction.") yield from self.get_table_upstream_workunits(discovered_tables) if self._external_lineage_map: # Some external lineage is yet to be emitted - yield from self.get_table_external_upstream_workunits(discovered_tables) + yield from self.get_table_external_upstream_workunits() - def get_table_external_upstream_workunits( - self, discovered_tables: List[str] - ) -> Iterable[MetadataWorkUnit]: + def get_table_external_upstream_workunits(self) -> Iterable[MetadataWorkUnit]: for ( dataset_name, external_lineage, @@ -133,7 +138,6 @@ def get_table_upstream_workunits( "Snowflake Account is Standard Edition. Table to Table and View to Table Lineage Feature is not supported." ) # See Edition Note above for why else: - results = None with PerfTimer() as timer: results = self._fetch_upstream_lineages_for_tables() self.report.table_lineage_query_secs = timer.elapsed_seconds() @@ -141,16 +145,30 @@ def get_table_upstream_workunits( if not results: return - yield from self._build_upstream_lineage_workunits_from_query_result( - discovered_tables, results - ) + yield from self._gen_workunits_from_query_result(discovered_tables, results) logger.info( f"Upstream lineage detected for {self.report.num_tables_with_upstreams} tables.", ) - def _build_upstream_lineage_workunits_from_query_result( - self, discovered_assets, results, upstream_for_view=False - ): + def _gen_workunit_from_sql_parsing_result( + self, + dataset_identifier: str, + result: SqlParsingResult, + ) -> MetadataWorkUnit: + upstreams, fine_upstreams = self.get_upstreams_from_sql_parsing_result( + self.dataset_urn_builder(dataset_identifier), result + ) + self.report.num_views_with_upstreams += 1 + return self._create_upstream_lineage_workunit( + dataset_identifier, upstreams, fine_upstreams + ) + + def _gen_workunits_from_query_result( + self, + discovered_assets: Collection[str], + results: Iterable[dict], + upstream_for_view: bool = False, + ) -> Iterable[MetadataWorkUnit]: for db_row in results: dataset_name = self.get_dataset_identifier_from_qualified_name( db_row["DOWNSTREAM_TABLE_NAME"] @@ -173,26 +191,70 @@ def _build_upstream_lineage_workunits_from_query_result( logger.debug(f"No lineage found for {dataset_name}") def get_view_upstream_workunits( - self, discovered_views: List[str] + self, + discovered_views: List[str], + schema_resolver: SchemaResolver, + view_definitions: MutableMapping[str, str], ) -> Iterable[MetadataWorkUnit]: - results = None + views_processed = set() + if self.config.include_view_column_lineage: + with PerfTimer() as timer: + for view_identifier, view_definition in view_definitions.items(): + result = self._run_sql_parser( + view_identifier, view_definition, schema_resolver + ) + if result: + views_processed.add(view_identifier) + yield self._gen_workunit_from_sql_parsing_result( + view_identifier, result + ) + self.report.view_lineage_parse_secs = timer.elapsed_seconds() + with PerfTimer() as timer: results = self._fetch_upstream_lineages_for_views() self.report.view_upstream_lineage_query_secs = timer.elapsed_seconds() - if not results: - return - - yield from self._build_upstream_lineage_workunits_from_query_result( - discovered_views, results, upstream_for_view=True - ) + if results: + yield from self._gen_workunits_from_query_result( + set(discovered_views) - views_processed, results, upstream_for_view=True + ) logger.info( f"Upstream lineage detected for {self.report.num_views_with_upstreams} views.", ) + def _run_sql_parser( + self, dataset_identifier: str, query: str, schema_resolver: SchemaResolver + ) -> Optional[SqlParsingResult]: + try: + database, schema, _view = dataset_identifier.split(".") + except ValueError: + logger.warning(f"Invalid view identifier: {dataset_identifier}") + return None + raw_lineage = sqlglot_lineage( + query, + schema_resolver=schema_resolver, + default_db=database, + default_schema=schema, + ) + if raw_lineage.debug_info.table_error: + logger.debug( + f"Failed to parse lineage for view {dataset_identifier}: " + f"{raw_lineage.debug_info.table_error}" + ) + self.report.num_view_definitions_failed_parsing += 1 + return None + elif raw_lineage.debug_info.column_error: + self.report.num_view_definitions_failed_column_parsing += 1 + else: + self.report.num_view_definitions_parsed += 1 + return raw_lineage + def _create_upstream_lineage_workunit( - self, dataset_name, upstreams, fine_upstreams=[] - ): + self, + dataset_name: str, + upstreams: Sequence[UpstreamClass], + fine_upstreams: Sequence[FineGrainedLineage] = (), + ) -> MetadataWorkUnit: logger.debug( f"Upstream lineage of '{dataset_name}': {[u.dataset for u in upstreams]}" ) @@ -200,7 +262,7 @@ def _create_upstream_lineage_workunit( self.report.upstream_lineage[dataset_name] = [u.dataset for u in upstreams] upstream_lineage = UpstreamLineage( - upstreams=upstreams, + upstreams=sorted(upstreams, key=lambda x: x.dataset), fineGrainedLineages=sorted( fine_upstreams, key=lambda x: (x.downstreams, x.upstreams), @@ -211,7 +273,9 @@ def _create_upstream_lineage_workunit( entityUrn=self.dataset_urn_builder(dataset_name), aspect=upstream_lineage ).as_workunit() - def get_upstreams_from_query_result_row(self, dataset_name, db_row): + def get_upstreams_from_query_result_row( + self, dataset_name: str, db_row: dict + ) -> Tuple[List[UpstreamClass], List[FineGrainedLineage]]: upstreams: List[UpstreamClass] = [] fine_upstreams: List[FineGrainedLineage] = [] @@ -237,6 +301,39 @@ def get_upstreams_from_query_result_row(self, dataset_name, db_row): return upstreams, fine_upstreams + def get_upstreams_from_sql_parsing_result( + self, downstream_table_urn: str, result: SqlParsingResult + ) -> Tuple[List[UpstreamClass], List[FineGrainedLineage]]: + # Note: This ignores the out_tables section of the sql parsing result. + upstreams = [ + UpstreamClass(dataset=upstream_table_urn, type=DatasetLineageTypeClass.VIEW) + for upstream_table_urn in set(result.in_tables) + ] + + # Maps downstream_col -> [upstream_col] + fine_lineage: Dict[str, Set[SnowflakeColumnId]] = defaultdict(set) + for column_lineage in result.column_lineage or []: + out_column = column_lineage.downstream.column + for upstream_column_info in column_lineage.upstreams: + upstream_table_name = DatasetUrn.create_from_string( + upstream_column_info.table + ).get_dataset_name() + fine_lineage[out_column].add( + SnowflakeColumnId( + columnName=upstream_column_info.column, + objectName=upstream_table_name, + objectDomain=SnowflakeObjectDomain.VIEW.value, + ) + ) + fine_upstreams = [ + self.build_finegrained_lineage( + downstream_table_urn, downstream_col, upstream_cols + ) + for downstream_col, upstream_cols in fine_lineage.items() + ] + + return upstreams, list(filter(None, fine_upstreams)) + def _populate_external_lineage_map(self, discovered_tables: List[str]) -> None: with PerfTimer() as timer: self.report.num_external_table_edges_scanned = 0 @@ -406,18 +503,14 @@ def _process_add_single_column_upstream( fine_upstream = self.build_finegrained_lineage( dataset_urn=dataset_urn, col=column_name, - fine_upstream=SnowflakeColumnFineGrainedLineage( - frozenset( - [ - SnowflakeColumnId( - columnName=col["column_name"], - objectName=col["object_name"], - objectDomain=col["object_domain"], - ) - for col in upstream_columns - ] + upstream_columns={ + SnowflakeColumnId( + columnName=col["column_name"], + objectName=col["object_name"], + objectDomain=col["object_domain"], ) - ), + for col in upstream_columns + }, ) if not fine_upstream: continue @@ -447,11 +540,9 @@ def build_finegrained_lineage( self, dataset_urn: str, col: str, - fine_upstream: SnowflakeColumnFineGrainedLineage, + upstream_columns: Set[SnowflakeColumnId], ) -> Optional[FineGrainedLineage]: - fieldPath = col - - column_upstreams = self.build_finegrained_lineage_upstreams(fine_upstream) + column_upstreams = self.build_finegrained_lineage_upstreams(upstream_columns) if not column_upstreams: return None finegrained_lineage_entry = FineGrainedLineage( @@ -462,7 +553,7 @@ def build_finegrained_lineage( downstreamType=FineGrainedLineageDownstreamType.FIELD, downstreams=[ builder.make_schema_field_urn( - dataset_urn, self.snowflake_identifier(fieldPath) + dataset_urn, self.snowflake_identifier(col) ) ], ) @@ -470,10 +561,10 @@ def build_finegrained_lineage( return finegrained_lineage_entry def build_finegrained_lineage_upstreams( - self, fine_upstream: SnowflakeColumnFineGrainedLineage + self, upstream_columms: Set[SnowflakeColumnId] ) -> List[str]: column_upstreams = [] - for upstream_col in fine_upstream.inputColumns: + for upstream_col in upstream_columms: if ( upstream_col.objectName and upstream_col.columnName diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py index 41616974a0b2d..8003de8286288 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py @@ -26,6 +26,7 @@ class SnowflakeV2Report( usage_aggregation_query_secs: float = -1 table_lineage_query_secs: float = -1 + view_lineage_parse_secs: float = -1 view_upstream_lineage_query_secs: float = -1 view_downstream_lineage_query_secs: float = -1 external_lineage_queries_secs: float = -1 @@ -59,6 +60,10 @@ class SnowflakeV2Report( num_tables_with_upstreams: int = 0 num_views_with_upstreams: int = 0 + num_view_definitions_parsed: int = 0 + num_view_definitions_failed_parsing: int = 0 + num_view_definitions_failed_column_parsing: int = 0 + def report_entity_scanned(self, name: str, ent_type: str = "table") -> None: """ Entity could be a view or a table or a schema or a database diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py index 2e37fb0de35b2..dab46645bffcc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py @@ -88,6 +88,7 @@ class SnowflakeTable(BaseTable): @dataclass class SnowflakeView(BaseView): + materialized: bool = False columns: List[SnowflakeColumn] = field(default_factory=list) tags: Optional[List[SnowflakeTag]] = None column_tags: Dict[str, List[SnowflakeTag]] = field(default_factory=dict) @@ -321,6 +322,8 @@ def get_views_for_database( comment=table["comment"], view_definition=table["text"], last_altered=table["created_on"], + materialized=table.get("is_materialized", "false").lower() + == "true", ) ) return views diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py index 4ac64b6106d89..bd3e5782ec2af 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py @@ -4,7 +4,7 @@ import os.path import platform from dataclasses import dataclass -from typing import Callable, Dict, Iterable, List, Optional, Union, cast +from typing import Callable, Dict, Iterable, List, Optional, Union import pandas as pd from snowflake.connector import SnowflakeConnection @@ -128,8 +128,10 @@ TimeType, ) from datahub.metadata.com.linkedin.pegasus2avro.tag import TagProperties +from datahub.utilities.file_backed_collections import FileBackedDict from datahub.utilities.perf_timer import PerfTimer from datahub.utilities.registries.domain_registry import DomainRegistry +from datahub.utilities.sqlglot_lineage import SchemaResolver from datahub.utilities.time import datetime_to_ts_millis logger: logging.Logger = logging.getLogger(__name__) @@ -284,6 +286,13 @@ def __init__(self, ctx: PipelineContext, config: SnowflakeV2Config): # Caches tables for a single database. Consider moving to disk or S3 when possible. self.db_tables: Dict[str, List[SnowflakeTable]] = {} + self.sql_parser_schema_resolver = SchemaResolver( + platform=self.platform, + platform_instance=self.config.platform_instance, + env=self.config.env, + ) + self.view_definitions: FileBackedDict[str] = FileBackedDict() + @classmethod def create(cls, config_dict: dict, ctx: PipelineContext) -> "Source": config = SnowflakeV2Config.parse_obj(config_dict) @@ -555,7 +564,10 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: if self.config.include_table_lineage: yield from self.lineage_extractor.get_workunits( - discovered_tables, discovered_views + discovered_tables=discovered_tables, + discovered_views=discovered_views, + schema_resolver=self.sql_parser_schema_resolver, + view_definitions=self.view_definitions, ) if self.config.include_usage_stats or self.config.include_operational_stats: @@ -747,14 +759,18 @@ def _process_schema( ) self.db_tables[schema_name] = tables - if self.config.include_technical_schema: + if self.config.include_technical_schema or self.config.parse_view_ddl: for table in tables: yield from self._process_table(table, schema_name, db_name) if self.config.include_views: views = self.fetch_views_for_schema(snowflake_schema, db_name, schema_name) + if self.config.parse_view_ddl: + for view in views: + key = self.get_dataset_identifier(view.name, schema_name, db_name) + self.view_definitions[key] = view.view_definition - if self.config.include_technical_schema: + if self.config.include_technical_schema or self.config.parse_view_ddl: for view in views: yield from self._process_view(view, schema_name, db_name) @@ -851,7 +867,9 @@ def _process_table( for tag in table.column_tags[column_name]: yield from self._process_tag(tag) - yield from self.gen_dataset_workunits(table, schema_name, db_name) + yield from self.gen_dataset_workunits(table, schema_name, db_name) + elif self.config.parse_view_ddl: + self.gen_schema_metadata(table, schema_name, db_name) def fetch_sample_data_for_classification( self, table, schema_name, db_name, dataset_name @@ -967,7 +985,9 @@ def _process_view( for tag in view.column_tags[column_name]: yield from self._process_tag(tag) - yield from self.gen_dataset_workunits(view, schema_name, db_name) + yield from self.gen_dataset_workunits(view, schema_name, db_name) + elif self.config.parse_view_ddl: + self.gen_schema_metadata(view, schema_name, db_name) def _process_tag(self, tag: SnowflakeTag) -> Iterable[MetadataWorkUnit]: tag_identifier = tag.identifier() @@ -1001,7 +1021,10 @@ def gen_dataset_workunits( entityUrn=dataset_urn, aspect=status ).as_workunit() - schema_metadata = self.get_schema_metadata(table, dataset_name, dataset_urn) + schema_metadata = self.gen_schema_metadata(table, schema_name, db_name) + # TODO: classification is only run for snowflake tables. + # Should we run classification for snowflake views as well? + self.classify_snowflake_table(table, dataset_name, schema_metadata) yield MetadataChangeProposalWrapper( entityUrn=dataset_urn, aspect=schema_metadata ).as_workunit() @@ -1064,15 +1087,11 @@ def gen_dataset_workunits( entityUrn=dataset_urn, aspect=global_tags ).as_workunit() - if ( - isinstance(table, SnowflakeView) - and cast(SnowflakeView, table).view_definition is not None - ): - view = cast(SnowflakeView, table) + if isinstance(table, SnowflakeView) and table.view_definition is not None: view_properties_aspect = ViewProperties( - materialized=False, + materialized=table.materialized, viewLanguage="SQL", - viewLogic=view.view_definition, + viewLogic=table.view_definition, ) yield MetadataChangeProposalWrapper( @@ -1117,12 +1136,15 @@ def gen_tag_workunits(self, tag: SnowflakeTag) -> Iterable[MetadataWorkUnit]: entityUrn=tag_urn, aspect=tag_properties_aspect ).as_workunit() - def get_schema_metadata( + def gen_schema_metadata( self, table: Union[SnowflakeTable, SnowflakeView], - dataset_name: str, - dataset_urn: str, + schema_name: str, + db_name: str, ) -> SchemaMetadata: + dataset_name = self.get_dataset_identifier(table.name, schema_name, db_name) + dataset_urn = self.gen_dataset_urn(dataset_name) + foreign_keys: Optional[List[ForeignKeyConstraint]] = None if isinstance(table, SnowflakeTable) and len(table.foreign_keys) > 0: foreign_keys = self.build_foreign_keys(table, dataset_urn, foreign_keys) @@ -1164,9 +1186,10 @@ def get_schema_metadata( foreignKeys=foreign_keys, ) - # TODO: classification is only run for snowflake tables. - # Should we run classification for snowflake views as well? - self.classify_snowflake_table(table, dataset_name, schema_metadata) + if self.config.parse_view_ddl: + self.sql_parser_schema_resolver.add_schema_metadata( + urn=dataset_urn, schema_metadata=schema_metadata + ) return schema_metadata @@ -1572,6 +1595,8 @@ def _snowflake_clear_ocsp_cache(self): def close(self) -> None: super().close() StatefulIngestionSourceBase.close(self) + self.view_definitions.close() + self.sql_parser_schema_resolver.close() if hasattr(self, "lineage_extractor"): self.lineage_extractor.close() if hasattr(self, "usage_extractor"): diff --git a/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py b/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py index b21ebe30c24a1..27ac2472bef93 100644 --- a/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py +++ b/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py @@ -91,7 +91,7 @@ class BaseSnowflakeConfig(BaseTimeWindowConfig): ) include_view_lineage: bool = pydantic.Field( default=True, - description="If enabled, populates the snowflake view->table and table->view lineages (no view->view lineage yet). Requires appropriate grants given to the role, and include_table_lineage to be True. view->table lineage requires Snowflake Enterprise Edition or above.", + description="If enabled, populates the snowflake view->table and table->view lineages. Requires appropriate grants given to the role, and include_table_lineage to be True. view->table lineage requires Snowflake Enterprise Edition or above.", ) connect_args: Optional[Dict[str, Any]] = pydantic.Field( default=None, diff --git a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py index bae97b96c4544..123a965e5a80a 100644 --- a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py +++ b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py @@ -20,6 +20,7 @@ DEFAULT_ENV, make_dataset_urn_with_platform_instance, ) +from datahub.ingestion.api.closeable import Closeable from datahub.ingestion.graph.client import DataHubGraph from datahub.ingestion.source.bigquery_v2.bigquery_audit import BigqueryTableIdentifier from datahub.metadata.schema_classes import SchemaMetadataClass @@ -223,7 +224,7 @@ def _table_level_lineage( return tables, modified -class SchemaResolver: +class SchemaResolver(Closeable): def __init__( self, *, @@ -340,6 +341,9 @@ def _convert_schema_aspect_to_info( # TODO add a method to load all from graphql + def close(self) -> None: + self._schema_cache.close() + # TODO: Once PEP 604 is supported (Python 3.10), we can unify these into a # single type. See https://peps.python.org/pep-0604/#isinstance-and-issubclass. From 20b3adb7b1a96f2a0101a50bce127aa20fd9bb72 Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Fri, 14 Jul 2023 21:31:24 -0400 Subject: [PATCH 147/222] fix(ingest/snowflake): Add sqlglot as snowflake dependency (#8427) --- metadata-ingestion/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index e8f233ca15ab3..2c7571ffca0d5 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -369,9 +369,9 @@ def get_long_description(): "gcs": {*s3_base, *data_lake_profiling}, "sagemaker": aws_common, "salesforce": {"simple-salesforce"}, - "snowflake": snowflake_common | usage_common, + "snowflake": snowflake_common | usage_common | sqlglot_lib, "snowflake-beta": ( - snowflake_common | usage_common + snowflake_common | usage_common | sqlglot_lib ), # deprecated, but keeping the extra for backwards compatibility "sqlalchemy": sql_common, "superset": { From e725d4df7447379cd72b0a59a19bef862b3b04be Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Mon, 17 Jul 2023 02:15:03 -0500 Subject: [PATCH 148/222] fix(schema-reg): allow other response codes from schema registry check (#8302) --- .../linkedin/metadata/boot/OnBootApplicationListener.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/OnBootApplicationListener.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/OnBootApplicationListener.java index dab5ad92677c5..0f52bc3816c2d 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/OnBootApplicationListener.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/OnBootApplicationListener.java @@ -2,6 +2,7 @@ import com.linkedin.gms.factory.config.ConfigurationProvider; import java.io.IOException; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import javax.annotation.Nonnull; @@ -25,6 +26,8 @@ @Slf4j @Component public class OnBootApplicationListener { + private static final Set ACCEPTED_HTTP_CODES = Set.of(HttpStatus.SC_OK, HttpStatus.SC_MOVED_PERMANENTLY, + HttpStatus.SC_MOVED_TEMPORARILY, HttpStatus.SC_FORBIDDEN, HttpStatus.SC_UNAUTHORIZED); private static final String ROOT_WEB_APPLICATION_CONTEXT_ID = String.format("%s:", WebApplicationContext.class.getName()); @@ -60,8 +63,8 @@ public Runnable isSchemaRegistryAPIServeletReady() { log.info("Sleeping for 1 second"); Thread.sleep(1000); StatusLine statusLine = httpClient.execute(request).getStatusLine(); - if (statusLine.getStatusCode() == HttpStatus.SC_OK) { - log.info("Connected!"); + if (ACCEPTED_HTTP_CODES.contains(statusLine.getStatusCode())) { + log.info("Connected! Authentication not tested."); openAPIServeletReady = true; } } catch (IOException | InterruptedException e) { From 9949f2ca6613cb841e7bb9e2ce8cfd8a1d0f3a8c Mon Sep 17 00:00:00 2001 From: Hyejin Yoon <0327jane@gmail.com> Date: Mon, 17 Jul 2023 17:29:00 +0900 Subject: [PATCH 149/222] fix: add docs on update description via graphQL (#8289) Co-authored-by: socar-dini Co-authored-by: Harshal Sheth --- docs/api/datahub-apis.md | 56 +++++++++++++------------- docs/api/tutorials/descriptions.md | 63 +++++++++++++++++++++++++++++- 2 files changed, 89 insertions(+), 30 deletions(-) diff --git a/docs/api/datahub-apis.md b/docs/api/datahub-apis.md index 62389a0072684..e9942cb19a4c9 100644 --- a/docs/api/datahub-apis.md +++ b/docs/api/datahub-apis.md @@ -61,31 +61,31 @@ Here's an overview of what each API can do. > Last Updated : Apr 8 2023 -| Feature | GraphQL | Python SDK | OpenAPI | -| ------------------------------------------------------- | --------------------------------------------------------------------------- | ---------------------------------------------------------------------------- | ------- | -| Create a dataset | 🚫 | ✅ [[Guide]](/docs/api/tutorials/datasets.md) | ✅ | -| Delete a dataset (Soft delete) | ✅ [[Guide]](/docs/api/tutorials/datasets.md#delete-dataset) | ✅ [[Guide]](/docs/api/tutorials/datasets.md#delete-dataset) | ✅ | -| Delete a dataset (Hard delele) | 🚫 | ✅ [[Guide]](/docs/api/tutorials/datasets.md#delete-dataset) | ✅ | -| Search a dataset | ✅ | ✅ | ✅ | -| Create a tag | ✅ [[Guide]](/docs/api/tutorials/tags.md) | ✅ [[Guide]](/docs/api/tutorials/tags.md) | ✅ | -| Read a tag | ✅ [[Guide]](/docs/api/tutorials/tags.md) | ✅ [[Guide]](/docs/api/tutorials/tags.md) | ✅ | -| Add tags to a dataset | ✅ [[Guide]](/docs/api/tutorials/tags.md) | ✅ [[Guide]](/docs/api/tutorials/tags.md) | ✅ | -| Add tags to a column of a dataset | ✅ [[Guide]](/docs/api/tutorials/tags.md) | ✅ [[Guide]](/docs/api/tutorials/tags.md) | ✅ | -| Remove tags from a dataset | ✅ [[Guide]](/docs/api/tutorials/tags.md) | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags) | ✅ | -| Create glossary terms | ✅ [[Guide]](/docs/api/tutorials/terms.md) | ✅ [[Guide]](/docs/api/tutorials/terms.md) | ✅ | -| Read terms from a dataset | ✅ [[Guide]](/docs/api/tutorials/terms.md) | ✅ [[Guide]](/docs/api/tutorials/terms.md) | ✅ | -| Add terms to a column of a dataset | ✅ [[Guide]](/docs/api/tutorials/terms.md) | ✅ [[Guide]](/docs/api/tutorials/terms.md) | ✅ | -| Add terms to a dataset | ✅ [[Guide]](/docs/api/tutorials/terms.md) | ✅ [[Guide]](/docs/api/tutorials/terms.md) | ✅ | -| Create domains | ✅ [[Guide]](/docs/api/tutorials/domains.md) | ✅ [[Guide]](/docs/api/tutorials/domains.md) | ✅ | -| Read domains | ✅ [[Guide]](/docs/api/tutorials/domains.md) | ✅ [[Guide]](/docs/api/tutorials/domains.md) | ✅ | -| Add domains to a dataset | ✅ [[Guide]](/docs/api/tutorials/domains.md) | ✅ [[Guide]](/docs/api/tutorials/domains.md) | ✅ | -| Remove domains from a dataset | ✅ [[Guide]](/docs/api/tutorials/domains.md) | ✅ [[Guide]](/docs/api/tutorials/domains.md) | ✅ | -| Crate users and groups | ✅ [[Guide]](/docs/api/tutorials/owners.md) | ✅ [[Guide]](/docs/api/tutorials/owners.md) | ✅ | -| Read owners of a dataset | ✅ [[Guide]](/docs/api/tutorials/owners.md) | ✅ [[Guide]](/docs/api/tutorials/owners.md) | ✅ | -| Add owner to a dataset | ✅ [[Guide]](/docs/api/tutorials/owners.md) | ✅ [[Guide]](/docs/api/tutorials/owners.md) | ✅ | -| Remove owner from a dataset | ✅ [[Guide]](/docs/api/tutorials/owners.md) | ✅ [[Guide]](/docs/api/tutorials/owners.md) | ✅ | -| Add lineage | ✅ [[Guide]](/docs/api/tutorials/lineage.md) | ✅ [[Guide]](/docs/api/tutorials/lineage.md) | ✅ | -| Add column level(Fine Grained) lineage | 🚫 | ✅ | ✅ | -| Add documentation(description) to a column of a dataset | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-column) | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-column) | ✅ | -| Add documentation(description) to a dataset | 🚫 | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-dataset) | ✅ | -| Add / Remove / Replace custom properties on a dataset | 🚫 [[Guide]](/docs/api/tutorials/custom-properties.md) | ✅ [[Guide]](/docs/api/tutorials/custom-properties.md) | ✅ | +| Feature | GraphQL | Python SDK | OpenAPI | +| ------------------------------------------------------- | ---------------------------------------------------------------------------- | ---------------------------------------------------------------------------- | ------- | +| Create a dataset | 🚫 | ✅ [[Guide]](/docs/api/tutorials/datasets.md) | ✅ | +| Delete a dataset (Soft delete) | ✅ [[Guide]](/docs/api/tutorials/datasets.md#delete-dataset) | ✅ [[Guide]](/docs/api/tutorials/datasets.md#delete-dataset) | ✅ | +| Delete a dataset (Hard delele) | 🚫 | ✅ [[Guide]](/docs/api/tutorials/datasets.md#delete-dataset) | ✅ | +| Search a dataset | ✅ | ✅ | ✅ | +| Create a tag | ✅ [[Guide]](/docs/api/tutorials/tags.md) | ✅ [[Guide]](/docs/api/tutorials/tags.md) | ✅ | +| Read a tag | ✅ [[Guide]](/docs/api/tutorials/tags.md) | ✅ [[Guide]](/docs/api/tutorials/tags.md) | ✅ | +| Add tags to a dataset | ✅ [[Guide]](/docs/api/tutorials/tags.md) | ✅ [[Guide]](/docs/api/tutorials/tags.md) | ✅ | +| Add tags to a column of a dataset | ✅ [[Guide]](/docs/api/tutorials/tags.md) | ✅ [[Guide]](/docs/api/tutorials/tags.md) | ✅ | +| Remove tags from a dataset | ✅ [[Guide]](/docs/api/tutorials/tags.md) | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags) | ✅ | +| Create glossary terms | ✅ [[Guide]](/docs/api/tutorials/terms.md) | ✅ [[Guide]](/docs/api/tutorials/terms.md) | ✅ | +| Read terms from a dataset | ✅ [[Guide]](/docs/api/tutorials/terms.md) | ✅ [[Guide]](/docs/api/tutorials/terms.md) | ✅ | +| Add terms to a column of a dataset | ✅ [[Guide]](/docs/api/tutorials/terms.md) | ✅ [[Guide]](/docs/api/tutorials/terms.md) | ✅ | +| Add terms to a dataset | ✅ [[Guide]](/docs/api/tutorials/terms.md) | ✅ [[Guide]](/docs/api/tutorials/terms.md) | ✅ | +| Create domains | ✅ [[Guide]](/docs/api/tutorials/domains.md) | ✅ [[Guide]](/docs/api/tutorials/domains.md) | ✅ | +| Read domains | ✅ [[Guide]](/docs/api/tutorials/domains.md) | ✅ [[Guide]](/docs/api/tutorials/domains.md) | ✅ | +| Add domains to a dataset | ✅ [[Guide]](/docs/api/tutorials/domains.md) | ✅ [[Guide]](/docs/api/tutorials/domains.md) | ✅ | +| Remove domains from a dataset | ✅ [[Guide]](/docs/api/tutorials/domains.md) | ✅ [[Guide]](/docs/api/tutorials/domains.md) | ✅ | +| Crate users and groups | ✅ [[Guide]](/docs/api/tutorials/owners.md) | ✅ [[Guide]](/docs/api/tutorials/owners.md) | ✅ | +| Read owners of a dataset | ✅ [[Guide]](/docs/api/tutorials/owners.md) | ✅ [[Guide]](/docs/api/tutorials/owners.md) | ✅ | +| Add owner to a dataset | ✅ [[Guide]](/docs/api/tutorials/owners.md) | ✅ [[Guide]](/docs/api/tutorials/owners.md) | ✅ | +| Remove owner from a dataset | ✅ [[Guide]](/docs/api/tutorials/owners.md) | ✅ [[Guide]](/docs/api/tutorials/owners.md) | ✅ | +| Add lineage | ✅ [[Guide]](/docs/api/tutorials/lineage.md) | ✅ [[Guide]](/docs/api/tutorials/lineage.md) | ✅ | +| Add column level(Fine Grained) lineage | 🚫 | ✅ | ✅ | +| Add documentation(description) to a column of a dataset | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-column) | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-column) | ✅ | +| Add documentation(description) to a dataset | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-dataset) | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-dataset) | ✅ | +| Add / Remove / Replace custom properties on a dataset | 🚫 [[Guide]](/docs/api/tutorials/custom-properties.md) | ✅ [[Guide]](/docs/api/tutorials/custom-properties.md) | ✅ | diff --git a/docs/api/tutorials/descriptions.md b/docs/api/tutorials/descriptions.md index 11c2b5cc65635..46f42b7a05be6 100644 --- a/docs/api/tutorials/descriptions.md +++ b/docs/api/tutorials/descriptions.md @@ -199,8 +199,67 @@ Expected Response: -> 🚫 Adding Description on Dataset via `graphql` is currently not supported. -> Please check out [API feature comparison table](/docs/api/datahub-apis.md#datahub-api-comparison) for more information, +```graphql +mutation updateDataset { + updateDataset( + urn:"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + input: { + editableProperties: { + description: "## The Real Estate Sales Dataset\nThis is a really important Dataset that contains all the relevant information about sales that have happened organized by address.\n" + } + institutionalMemory: { + elements: { + author: "urn:li:corpuser:jdoe" + url: "https://wikipedia.com/real_estate" + description: "This is the definition of what real estate means" + } + } + } + ) { + urn + } +} +``` + +Expected Response: + +```json +{ + "data": { + "updateDataset": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" + } + }, + "extensions": {} +} +``` + + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "query": "mutation updateDataset { updateDataset( urn:\"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)\", input: { editableProperties: { description: \"## The Real Estate Sales Dataset\nThis is a really important Dataset that contains all the relevant information about sales that have happened organized by address.\n\" } institutionalMemory: { elements: { author: \"urn:li:corpuser:jdoe\", url: \"https://wikipedia.com/real_estate\", description: \"This is the definition of what real estate means\" } } } ) { urn } }", + "variables": {} +}' +``` + +Expected Response: + +```json +{ + "data": { + "updateDataset": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" + } + }, + "extensions": {} +} +``` From 9cb62f30fd48cff537901f4d20f5291c0768f8a0 Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Mon, 17 Jul 2023 04:44:39 -0400 Subject: [PATCH 150/222] docs(databricks/spark-lineage): Fix incorrect statement (#8423) --- metadata-integration/java/spark-lineage/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata-integration/java/spark-lineage/README.md b/metadata-integration/java/spark-lineage/README.md index 599fe6435ce11..f56cb14a1ae54 100644 --- a/metadata-integration/java/spark-lineage/README.md +++ b/metadata-integration/java/spark-lineage/README.md @@ -83,8 +83,8 @@ The Spark agent can be configured using Databricks Cluster [Spark configuration] ```sh databricks fs mkdirs dbfs:/datahub - databricks fs --overwrite datahub-spark-lineage*.jar dbfs:/datahub - databricks fs --overwrite init.sh dbfs:/datahub + databricks fs cp --overwrite datahub-spark-lineage*.jar dbfs:/datahub + databricks fs cp --overwrite init.sh dbfs:/datahub ``` - Open Databricks Cluster configuration page. Click the **Advanced Options** toggle. Click the **Spark** tab. Add below configurations under `Spark Config`. From 620d23d623a49c2c9349e0e75f7a17228ba5ff3b Mon Sep 17 00:00:00 2001 From: Joshua Eilers Date: Mon, 17 Jul 2023 09:55:54 -0700 Subject: [PATCH 151/222] feat(browsev2): styling updates and select platform (#8428) --- datahub-web-react/src/app/analytics/event.ts | 2 +- .../src/app/search/SearchResultList.tsx | 1 + .../src/app/search/SearchResults.tsx | 32 ++++++--- .../src/app/search/sidebar/BrowseContext.tsx | 8 +-- .../src/app/search/sidebar/BrowseNode.tsx | 3 +- .../src/app/search/sidebar/BrowseSidebar.tsx | 1 + .../src/app/search/sidebar/PlatformNode.tsx | 69 +++++++++++-------- 7 files changed, 69 insertions(+), 47 deletions(-) diff --git a/datahub-web-react/src/app/analytics/event.ts b/datahub-web-react/src/app/analytics/event.ts index 76826a30ea3eb..84173b522fb07 100644 --- a/datahub-web-react/src/app/analytics/event.ts +++ b/datahub-web-react/src/app/analytics/event.ts @@ -245,7 +245,7 @@ export interface BrowseV2ToggleNodeEvent extends BaseEvent { */ export interface BrowseV2SelectNodeEvent extends BaseEvent { type: EventType.BrowseV2SelectNodeEvent; - targetNode: 'browse'; + targetNode: 'browse' | 'platform'; action: 'select' | 'deselect'; entity: string; environment?: string; diff --git a/datahub-web-react/src/app/search/SearchResultList.tsx b/datahub-web-react/src/app/search/SearchResultList.tsx index 591927cfb152d..a793348db6f1d 100644 --- a/datahub-web-react/src/app/search/SearchResultList.tsx +++ b/datahub-web-react/src/app/search/SearchResultList.tsx @@ -48,6 +48,7 @@ const ResultWrapper = styled.div<{ showUpdatedStyles: boolean }>` margin: 0 auto 8px auto; padding: 8px 16px; max-width: 1200px; + border-bottom: 1px solid ${ANTD_GRAY[5]}; `} `; diff --git a/datahub-web-react/src/app/search/SearchResults.tsx b/datahub-web-react/src/app/search/SearchResults.tsx index 8a0e31e1fb8a0..61d5ba3cd98cd 100644 --- a/datahub-web-react/src/app/search/SearchResults.tsx +++ b/datahub-web-react/src/app/search/SearchResults.tsx @@ -26,12 +26,12 @@ import { BrowseProvider } from './sidebar/BrowseContext'; import { useIsBrowseV2, useIsSearchV2 } from './useSearchAndBrowseVersion'; import useToggleSidebar from './useToggleSidebar'; -const SearchResultsWrapper = styled.div<{ showUpdatedStyles: boolean }>` +const SearchResultsWrapper = styled.div<{ v2Styles: boolean }>` display: flex; flex: 1; ${(props) => - props.showUpdatedStyles && + props.v2Styles && ` overflow: hidden; `} @@ -45,12 +45,14 @@ const SearchBody = styled.div` overflow: auto; `; -const ResultContainer = styled.div<{ displayUpdatedStyles: boolean }>` +const ResultContainer = styled.div<{ v2Styles: boolean }>` flex: 1; overflow: auto; ${(props) => - props.displayUpdatedStyles + props.v2Styles ? ` + display: flex; + flex-direction: column; background-color: #F8F9FA; ` : ` @@ -64,7 +66,7 @@ const PaginationControlContainer = styled.div` text-align: center; `; -const PaginationInfoContainer = styled.div` +const PaginationInfoContainer = styled.div<{ v2Styles: boolean }>` padding-left: 24px; padding-right: 32px; height: 47px; @@ -73,6 +75,7 @@ const PaginationInfoContainer = styled.div` display: flex; justify-content: space-between; align-items: center; + ${({ v2Styles }) => v2Styles && `background-color: white;`} `; const LeftControlsContainer = styled.div` @@ -92,6 +95,15 @@ const StyledTabToolbar = styled(TabToolbar)` const SearchMenuContainer = styled.div``; +const SearchResultListContainer = styled.div<{ v2Styles: boolean }>` + ${({ v2Styles }) => + v2Styles && + ` + flex: 1; + overflow: auto; + `} +`; + interface Props { unionType?: UnionType; query: string; @@ -163,7 +175,7 @@ export const SearchResults = ({ return ( <> {loading && } - + {!showSearchFiltersV2 && (
@@ -184,8 +196,8 @@ export const SearchResults = ({ )} - - + + {showBrowseV2 && } @@ -223,7 +235,7 @@ export const SearchResults = ({ )} {(error && ) || (!loading && ( - <> + )} - + ))} diff --git a/datahub-web-react/src/app/search/sidebar/BrowseContext.tsx b/datahub-web-react/src/app/search/sidebar/BrowseContext.tsx index 3f96f3e844b5d..487ff8715d5bb 100644 --- a/datahub-web-react/src/app/search/sidebar/BrowseContext.tsx +++ b/datahub-web-react/src/app/search/sidebar/BrowseContext.tsx @@ -185,7 +185,7 @@ export const useOnSelectBrowsePath = () => { const selectedFilters = useSelectedFilters(); const onChangeFilters = useOnChangeFilters(); - return (isSelected: boolean) => { + return (isSelected: boolean, removeFilters: string[] = []) => { const overrides: Array = []; // keep entity and subType filters for this given entity only if they exist, otherwise apply this entity filter @@ -215,9 +215,9 @@ export const useOnSelectBrowsePath = () => { values: [browseSearchFilter], }); - const filtersWithOverrides = applyFacetFilterOverrides(selectedFilters, overrides).filter( - (filter) => isSelected || !overrides.some((override) => override.field === filter.field), - ); + const filtersWithOverrides = applyFacetFilterOverrides(selectedFilters, overrides) + .filter((filter) => !removeFilters.includes(filter.field)) + .filter((filter) => isSelected || !overrides.some((override) => override.field === filter.field)); onChangeFilters(filtersWithOverrides); }; diff --git a/datahub-web-react/src/app/search/sidebar/BrowseNode.tsx b/datahub-web-react/src/app/search/sidebar/BrowseNode.tsx index fefe1f8fc88cd..ab066027eff52 100644 --- a/datahub-web-react/src/app/search/sidebar/BrowseNode.tsx +++ b/datahub-web-react/src/app/search/sidebar/BrowseNode.tsx @@ -41,11 +41,10 @@ const BrowseNode = () => { const environmentAggregation = useMaybeEnvironmentAggregation(); const platformAggregation = usePlatformAggregation(); const browseResultGroup = useBrowseResultGroup(); - const { trackToggleNodeEvent } = useSidebarAnalytics(); const { count, entity } = browseResultGroup; const hasEntityLink = !!entity; const displayName = useBrowseDisplayName(); - const { trackSelectNodeEvent } = useSidebarAnalytics(); + const { trackSelectNodeEvent, trackToggleNodeEvent } = useSidebarAnalytics(); const { isOpen, isClosing, toggle } = useToggle({ initialValue: isBrowsePathPrefix && !isBrowsePathSelected, diff --git a/datahub-web-react/src/app/search/sidebar/BrowseSidebar.tsx b/datahub-web-react/src/app/search/sidebar/BrowseSidebar.tsx index 64aa4e70cb117..b5e9272cc5273 100644 --- a/datahub-web-react/src/app/search/sidebar/BrowseSidebar.tsx +++ b/datahub-web-react/src/app/search/sidebar/BrowseSidebar.tsx @@ -13,6 +13,7 @@ const Sidebar = styled.div<{ visible: boolean; width: number }>` transition: width 250ms ease-in-out; border-right: 1px solid ${(props) => props.theme.styles['border-color-base']}; background-color: #f8f9fa; + background: white; `; const SidebarHeader = styled.div` diff --git a/datahub-web-react/src/app/search/sidebar/PlatformNode.tsx b/datahub-web-react/src/app/search/sidebar/PlatformNode.tsx index 644ba2f9caffe..dad7d8ac02bea 100644 --- a/datahub-web-react/src/app/search/sidebar/PlatformNode.tsx +++ b/datahub-web-react/src/app/search/sidebar/PlatformNode.tsx @@ -5,7 +5,7 @@ import { formatNumber } from '../../shared/formatNumber'; import ExpandableNode from './ExpandableNode'; import { useEntityRegistry } from '../../useEntityRegistry'; import { getFilterIconAndLabel } from '../filters/utils'; -import { PLATFORM_FILTER_NAME } from '../utils/constants'; +import { BROWSE_PATH_V2_FILTER_NAME, PLATFORM_FILTER_NAME } from '../utils/constants'; import useBrowsePagination from './useBrowsePagination'; import BrowseNode from './BrowseNode'; import SidebarLoadingError from './SidebarLoadingError'; @@ -15,9 +15,11 @@ import { useEntityAggregation, useIsPlatformSelected, useMaybeEnvironmentAggregation, + useOnSelectBrowsePath, usePlatformAggregation, } from './BrowseContext'; import useSidebarAnalytics from './useSidebarAnalytics'; +import { useHasFilterField } from './SidebarContext'; const PlatformIconContainer = styled.div` width: 16px; @@ -31,20 +33,18 @@ const Count = styled(Typography.Text)` padding-right: 8px; `; -const BrowseGroupListContainer = styled.div` - background: white; - border-radius: 8px; - padding-right: 8px; -`; - const PlatformNode = () => { - const isSelected = useIsPlatformSelected(); + const isPlatformSelected = useIsPlatformSelected(); + const hasBrowseFilter = useHasFilterField(BROWSE_PATH_V2_FILTER_NAME); + const isPlatformAndPathSelected = isPlatformSelected && hasBrowseFilter; + const isPlatformOnlySelected = isPlatformSelected && !hasBrowseFilter; const entityAggregation = useEntityAggregation(); const environmentAggregation = useMaybeEnvironmentAggregation(); const platformAggregation = usePlatformAggregation(); const { count } = platformAggregation; const registry = useEntityRegistry(); - const { trackToggleNodeEvent } = useSidebarAnalytics(); + const { trackToggleNodeEvent, trackSelectNodeEvent } = useSidebarAnalytics(); + const onSelectBrowsePath = useOnSelectBrowsePath(); const { icon, label } = getFilterIconAndLabel( PLATFORM_FILTER_NAME, @@ -55,15 +55,21 @@ const PlatformNode = () => { ); const { isOpen, isClosing, toggle } = useToggle({ - initialValue: isSelected, + initialValue: isPlatformAndPathSelected, closeDelay: 250, onToggle: (isNowOpen: boolean) => trackToggleNodeEvent(isNowOpen, 'platform'), }); - const onClickHeader = () => { + const onClickTriangle = () => { if (count) toggle(); }; + const onClickHeader = () => { + const isNowPlatformOnlySelected = !isPlatformOnlySelected; + onSelectBrowsePath(isNowPlatformOnlySelected, [BROWSE_PATH_V2_FILTER_NAME]); + trackSelectNodeEvent(isNowPlatformOnlySelected ? 'select' : 'deselect', 'platform'); + }; + const { error, groups, loaded, observable, path, retry } = useBrowsePagination({ skip: !isOpen }); const color = '#000'; @@ -72,12 +78,17 @@ const PlatformNode = () => { + {icon} @@ -86,26 +97,24 @@ const PlatformNode = () => { {formatNumber(platformAggregation.count)} - + } body={ - - {groups.map((group) => ( - - - - ))} - {error && } - {observable} - + {groups.map((group) => ( + + + + ))} + {error && } + {observable} } /> From cf8e44b69ab27f2101f01dbea0c697cdd8abdac4 Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Mon, 17 Jul 2023 13:22:37 -0700 Subject: [PATCH 152/222] fix(ui ingestion): fixing issue where stale fields could stick around when changing recipes (#8421) --- .../src/app/ingest/source/builder/DefineRecipeStep.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/datahub-web-react/src/app/ingest/source/builder/DefineRecipeStep.tsx b/datahub-web-react/src/app/ingest/source/builder/DefineRecipeStep.tsx index 46b5ec0bcd120..4ff4623b548c9 100644 --- a/datahub-web-react/src/app/ingest/source/builder/DefineRecipeStep.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/DefineRecipeStep.tsx @@ -45,12 +45,14 @@ export const DefineRecipeStep = ({ state, updateState, goTo, prev, ingestionSour const placeholderRecipe = getPlaceholderRecipe(ingestionSources, type); const [stagedRecipeYml, setStagedRecipeYml] = useState(existingRecipeYaml || placeholderRecipe); + const [stagedRecipeName, setStagedRecipeName] = useState(state.name); useEffect(() => { if (existingRecipeYaml) { + setStagedRecipeName(state.name); setStagedRecipeYml(existingRecipeYaml); } - }, [existingRecipeYaml]); + }, [existingRecipeYaml, state.name]); const [stepComplete, setStepComplete] = useState(false); @@ -97,6 +99,7 @@ export const DefineRecipeStep = ({ state, updateState, goTo, prev, ingestionSour if (type && CONNECTORS_WITH_FORM.has(type)) { return ( Date: Mon, 17 Jul 2023 15:26:43 -0700 Subject: [PATCH 153/222] ci: workarounds for pyyaml installation (#8435) --- docker/kafka-setup/Dockerfile | 8 +++++--- docker/quickstart/generate_and_compare.sh | 1 + docker/quickstart/requirements.txt | 2 +- metadata-ingestion-modules/airflow-plugin/build.gradle | 4 ++++ 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docker/kafka-setup/Dockerfile b/docker/kafka-setup/Dockerfile index e379ae4385246..8cf9d0869dc9b 100644 --- a/docker/kafka-setup/Dockerfile +++ b/docker/kafka-setup/Dockerfile @@ -5,7 +5,7 @@ ARG MAVEN_REPO="https://repo1.maven.org/maven2" ARG SNAKEYAML_VERSION="2.0" RUN rm /usr/share/java/cp-base-new/snakeyaml-*.jar \ - && wget -P /usr/share/java/cp-base-new $MAVEN_REPO/org/yaml/snakeyaml/$SNAKEYAML_VERSION/snakeyaml-$SNAKEYAML_VERSION.jar + && wget -P /usr/share/java/cp-base-new $MAVEN_REPO/org/yaml/snakeyaml/$SNAKEYAML_VERSION/snakeyaml-$SNAKEYAML_VERSION.jar # Based on https://github.com/blacktop's alpine kafka build FROM python:3-alpine @@ -17,7 +17,7 @@ ENV SCALA_VERSION 2.13 ENV CUB_CLASSPATH='"/usr/share/java/cp-base-new/*"' # Confluent Docker Utils Version (Namely the tag or branch to grab from git to install) -ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="v0.0.58" +ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="v0.0.60" # This can be overriden for an offline/air-gapped builds ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluentinc/confluent-docker-utils@${PYTHON_CONFLUENT_DOCKER_UTILS_VERSION}" @@ -36,7 +36,9 @@ RUN mkdir -p /opt \ && adduser -DH -s /sbin/nologin kafka \ && chown -R kafka: /opt/kafka \ && echo "===> Installing python packages ..." \ - && pip install --no-cache-dir jinja2 requests \ + && pip install --no-cache-dir --upgrade pip wheel setuptools \ + && pip install jinja2 requests \ + && pip install "Cython<3.0" "PyYAML<6" --no-build-isolation \ && pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ && rm -rf /tmp/* \ && apk del --purge .build-deps diff --git a/docker/quickstart/generate_and_compare.sh b/docker/quickstart/generate_and_compare.sh index e34abeb9820c6..d568eb3a4c246 100755 --- a/docker/quickstart/generate_and_compare.sh +++ b/docker/quickstart/generate_and_compare.sh @@ -8,5 +8,6 @@ set -euxo pipefail python3 -m venv venv source venv/bin/activate +pip install --upgrade pip wheel setuptools pip install -r requirements.txt python generate_docker_quickstart.py check-all diff --git a/docker/quickstart/requirements.txt b/docker/quickstart/requirements.txt index a20e96afc8582..539241331120c 100644 --- a/docker/quickstart/requirements.txt +++ b/docker/quickstart/requirements.txt @@ -1,3 +1,3 @@ -PyYAML==5.4.1 +PyYAML==6.0 python-dotenv==0.17.0 click diff --git a/metadata-ingestion-modules/airflow-plugin/build.gradle b/metadata-ingestion-modules/airflow-plugin/build.gradle index 9ab590b1560b3..d895e29229c41 100644 --- a/metadata-ingestion-modules/airflow-plugin/build.gradle +++ b/metadata-ingestion-modules/airflow-plugin/build.gradle @@ -22,6 +22,10 @@ task environmentSetup(type: Exec, dependsOn: checkPythonVersion) { task installPackage(type: Exec, dependsOn: environmentSetup) { inputs.file file('setup.py') outputs.dir("${venv_name}") + // Workaround for https://github.com/yaml/pyyaml/issues/601. + // See https://github.com/yaml/pyyaml/issues/601#issuecomment-1638509577. + // and https://github.com/datahub-project/datahub/pull/8435. + commandLine 'bash', '-x', '-c', "${pip_install_command} install 'Cython<3.0' 'PyYAML<6' --no-build-isolation" commandLine 'bash', '-x', '-c', "${pip_install_command} -e ." } From 48c1dc820ec472fb39859d30cb8e886f99808d70 Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Tue, 18 Jul 2023 17:29:50 -0400 Subject: [PATCH 154/222] build(ingest/boto3): Update boto3-stubs to fix CI (#8452) --- metadata-ingestion/setup.py | 2 +- .../sagemaker_processors/feature_groups.py | 12 +++---- .../aws/sagemaker_processors/lineage.py | 22 +++++------- .../source/aws/sagemaker_processors/models.py | 34 +++++++++---------- 4 files changed, 33 insertions(+), 37 deletions(-) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 2c7571ffca0d5..029527ea959d5 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -413,7 +413,7 @@ def get_long_description(): "types-cachetools", # versions 0.1.13 and 0.1.14 seem to have issues "types-click==0.1.12", - "boto3-stubs[s3,glue,sagemaker,sts]>=1.28.3", + "boto3-stubs[s3,glue,sagemaker,sts]>=1.28.4", "types-tabulate", # avrogen package requires this "types-pytz", diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py index 207b73d4cbd46..75fae47c966fd 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py @@ -23,9 +23,9 @@ if TYPE_CHECKING: from mypy_boto3_sagemaker import SageMakerClient from mypy_boto3_sagemaker.type_defs import ( - DescribeFeatureGroupResponseOutputTypeDef, + DescribeFeatureGroupResponseTypeDef, FeatureDefinitionOutputTypeDef, - FeatureGroupSummaryOutputTypeDef, + FeatureGroupSummaryTypeDef, ) @@ -35,7 +35,7 @@ class FeatureGroupProcessor: env: str report: SagemakerSourceReport - def get_all_feature_groups(self) -> List["FeatureGroupSummaryOutputTypeDef"]: + def get_all_feature_groups(self) -> List["FeatureGroupSummaryTypeDef"]: """ List all feature groups in SageMaker. """ @@ -51,7 +51,7 @@ def get_all_feature_groups(self) -> List["FeatureGroupSummaryOutputTypeDef"]: def get_feature_group_details( self, feature_group_name: str - ) -> "DescribeFeatureGroupResponseOutputTypeDef": + ) -> "DescribeFeatureGroupResponseTypeDef": """ Get details of a feature group (including list of component features). """ @@ -75,7 +75,7 @@ def get_feature_group_details( return feature_group def get_feature_group_wu( - self, feature_group_details: "DescribeFeatureGroupResponseOutputTypeDef" + self, feature_group_details: "DescribeFeatureGroupResponseTypeDef" ) -> MetadataWorkUnit: """ Generate an MLFeatureTable workunit for a SageMaker feature group. @@ -146,7 +146,7 @@ def get_feature_type(self, aws_type: str, feature_name: str) -> str: def get_feature_wu( self, - feature_group_details: "DescribeFeatureGroupResponseOutputTypeDef", + feature_group_details: "DescribeFeatureGroupResponseTypeDef", feature: "FeatureDefinitionOutputTypeDef", ) -> MetadataWorkUnit: """ diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/lineage.py index 7ba615d80346f..b677dccad24ac 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/lineage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/lineage.py @@ -9,10 +9,10 @@ if TYPE_CHECKING: from mypy_boto3_sagemaker import SageMakerClient from mypy_boto3_sagemaker.type_defs import ( - ActionSummaryOutputTypeDef, - ArtifactSummaryOutputTypeDef, - AssociationSummaryOutputTypeDef, - ContextSummaryOutputTypeDef, + ActionSummaryTypeDef, + ArtifactSummaryTypeDef, + AssociationSummaryTypeDef, + ContextSummaryTypeDef, ) @@ -49,7 +49,7 @@ class LineageProcessor: nodes: Dict[str, Dict[str, Any]] = field(default_factory=dict) lineage_info: LineageInfo = field(default_factory=LineageInfo) - def get_all_actions(self) -> List["ActionSummaryOutputTypeDef"]: + def get_all_actions(self) -> List["ActionSummaryTypeDef"]: """ List all actions in SageMaker. """ @@ -63,7 +63,7 @@ def get_all_actions(self) -> List["ActionSummaryOutputTypeDef"]: return actions - def get_all_artifacts(self) -> List["ArtifactSummaryOutputTypeDef"]: + def get_all_artifacts(self) -> List["ArtifactSummaryTypeDef"]: """ List all artifacts in SageMaker. """ @@ -77,7 +77,7 @@ def get_all_artifacts(self) -> List["ArtifactSummaryOutputTypeDef"]: return artifacts - def get_all_contexts(self) -> List["ContextSummaryOutputTypeDef"]: + def get_all_contexts(self) -> List["ContextSummaryTypeDef"]: """ List all contexts in SageMaker. """ @@ -91,9 +91,7 @@ def get_all_contexts(self) -> List["ContextSummaryOutputTypeDef"]: return contexts - def get_incoming_edges( - self, node_arn: str - ) -> List["AssociationSummaryOutputTypeDef"]: + def get_incoming_edges(self, node_arn: str) -> List["AssociationSummaryTypeDef"]: """ Get all incoming edges for a node in the lineage graph. """ @@ -107,9 +105,7 @@ def get_incoming_edges( return edges - def get_outgoing_edges( - self, node_arn: str - ) -> List["AssociationSummaryOutputTypeDef"]: + def get_outgoing_edges(self, node_arn: str) -> List["AssociationSummaryTypeDef"]: """ Get all outgoing edges for a node in the lineage graph. """ diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/models.py b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/models.py index 12770b0831693..e82cfc58f75a7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/models.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/models.py @@ -45,12 +45,12 @@ if TYPE_CHECKING: from mypy_boto3_sagemaker import SageMakerClient from mypy_boto3_sagemaker.type_defs import ( - DescribeEndpointOutputOutputTypeDef, - DescribeModelOutputOutputTypeDef, - DescribeModelPackageGroupOutputOutputTypeDef, - EndpointSummaryOutputTypeDef, - ModelPackageGroupSummaryOutputTypeDef, - ModelSummaryOutputTypeDef, + DescribeEndpointOutputTypeDef, + DescribeModelOutputTypeDef, + DescribeModelPackageGroupOutputTypeDef, + EndpointSummaryTypeDef, + ModelPackageGroupSummaryTypeDef, + ModelSummaryTypeDef, ) ENDPOINT_STATUS_MAP: Dict[str, str] = { @@ -91,7 +91,7 @@ class ModelProcessor: group_arn_to_name: Dict[str, str] = field(default_factory=dict) - def get_all_models(self) -> List["ModelSummaryOutputTypeDef"]: + def get_all_models(self) -> List["ModelSummaryTypeDef"]: """ List all models in SageMaker. """ @@ -105,7 +105,7 @@ def get_all_models(self) -> List["ModelSummaryOutputTypeDef"]: return models - def get_model_details(self, model_name: str) -> "DescribeModelOutputOutputTypeDef": + def get_model_details(self, model_name: str) -> "DescribeModelOutputTypeDef": """ Get details of a model. """ @@ -113,7 +113,7 @@ def get_model_details(self, model_name: str) -> "DescribeModelOutputOutputTypeDe # see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sagemaker.html#SageMaker.Client.describe_model return self.sagemaker_client.describe_model(ModelName=model_name) - def get_all_groups(self) -> List["ModelPackageGroupSummaryOutputTypeDef"]: + def get_all_groups(self) -> List["ModelPackageGroupSummaryTypeDef"]: """ List all model groups in SageMaker. """ @@ -128,7 +128,7 @@ def get_all_groups(self) -> List["ModelPackageGroupSummaryOutputTypeDef"]: def get_group_details( self, group_name: str - ) -> "DescribeModelPackageGroupOutputOutputTypeDef": + ) -> "DescribeModelPackageGroupOutputTypeDef": """ Get details of a model group. """ @@ -138,7 +138,7 @@ def get_group_details( ModelPackageGroupName=group_name ) - def get_all_endpoints(self) -> List["EndpointSummaryOutputTypeDef"]: + def get_all_endpoints(self) -> List["EndpointSummaryTypeDef"]: endpoints = [] # see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sagemaker.html#SageMaker.Client.list_endpoints @@ -151,7 +151,7 @@ def get_all_endpoints(self) -> List["EndpointSummaryOutputTypeDef"]: def get_endpoint_details( self, endpoint_name: str - ) -> "DescribeEndpointOutputOutputTypeDef": + ) -> "DescribeEndpointOutputTypeDef": # see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sagemaker.html#SageMaker.Client.describe_endpoint return self.sagemaker_client.describe_endpoint(EndpointName=endpoint_name) @@ -171,7 +171,7 @@ def get_endpoint_status( return endpoint_status def get_endpoint_wu( - self, endpoint_details: "DescribeEndpointOutputOutputTypeDef" + self, endpoint_details: "DescribeEndpointOutputTypeDef" ) -> MetadataWorkUnit: """a Get a workunit for an endpoint. @@ -215,7 +215,7 @@ def get_endpoint_wu( def get_model_endpoints( self, - model_details: "DescribeModelOutputOutputTypeDef", + model_details: "DescribeModelOutputTypeDef", endpoint_arn_to_name: Dict[str, str], model_image: Optional[str], model_uri: Optional[str], @@ -244,7 +244,7 @@ def get_model_endpoints( return model_endpoints_sorted def get_group_wu( - self, group_details: "DescribeModelPackageGroupOutputOutputTypeDef" + self, group_details: "DescribeModelPackageGroupOutputTypeDef" ) -> MetadataWorkUnit: """ Get a workunit for a model group. @@ -294,7 +294,7 @@ def get_group_wu( return MetadataWorkUnit(id=group_name, mce=mce) def match_model_jobs( - self, model_details: "DescribeModelOutputOutputTypeDef" + self, model_details: "DescribeModelOutputTypeDef" ) -> Tuple[Set[str], Set[str], List[MLHyperParamClass], List[MLMetricClass]]: model_training_jobs: Set[str] = set() model_downstream_jobs: Set[str] = set() @@ -387,7 +387,7 @@ def strip_quotes(string: str) -> str: def get_model_wu( self, - model_details: "DescribeModelOutputOutputTypeDef", + model_details: "DescribeModelOutputTypeDef", endpoint_arn_to_name: Dict[str, str], ) -> MetadataWorkUnit: """ From cc2dc342c6164b0a1d332d45acdc68d31ba843a6 Mon Sep 17 00:00:00 2001 From: Arun Vasudevan <12974850+arunvasudevan@users.noreply.github.com> Date: Tue, 18 Jul 2023 21:24:54 -0500 Subject: [PATCH 155/222] fix(ingestion-redshift): Fix Redshift ingestion logs (#8454) --- .../src/datahub/ingestion/source/redshift/redshift.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py index 099e982691c67..c302497a48101 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py @@ -813,7 +813,7 @@ def cache_tables_and_views(self, connection, database): ) else: logger.debug( - f"View {database}.{schema}.{table.name} is filtered by view_pattern" + f"View {database}.{schema}.{view.name} is filtered by view_pattern" ) self.report.view_filtered[f"{database}.{schema}"] = ( self.report.view_filtered.get(f"{database}.{schema}", 0) + 1 From addf76c8494fe08559ff000358b993a797526e9b Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 19 Jul 2023 03:36:42 -0700 Subject: [PATCH 156/222] fix(ingest/bigquery): make sql parsing more robust (#8450) --- .../ingestion/source/bigquery_v2/bigquery.py | 6 ++++++ .../source/bigquery_v2/bigquery_report.py | 5 +++-- .../ingestion/source/bigquery_v2/lineage.py | 15 ++++++++++----- .../src/datahub/utilities/stats_collections.py | 9 +++++++-- 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py index 1e4a293239942..919c803222066 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py @@ -689,9 +689,15 @@ def generate_lineage(self, project_id: str) -> Iterable[MetadataWorkUnit]: f"Failed to parse lineage for view {view}: {raw_view_lineage.debug_info.table_error}" ) self.report.num_view_definitions_failed_parsing += 1 + self.report.view_definitions_parsing_failures.append( + f"Table-level sql parsing error for view {view}: {raw_view_lineage.debug_info.table_error}" + ) continue elif raw_view_lineage.debug_info.column_error: self.report.num_view_definitions_failed_column_parsing += 1 + self.report.view_definitions_parsing_failures.append( + f"Column-level sql parsing error for view {view}: {raw_view_lineage.debug_info.column_error}" + ) else: self.report.num_view_definitions_parsed += 1 diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py index b86f5cabc6b14..b57e691411f75 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py @@ -79,6 +79,7 @@ class BigQueryV2Report(ProfilingSqlReport): num_view_definitions_parsed: int = 0 num_view_definitions_failed_parsing: int = 0 num_view_definitions_failed_column_parsing: int = 0 + view_definitions_parsing_failures: LossyList[str] = field(default_factory=LossyList) read_reasons_stat: Counter[str] = dataclasses.field( default_factory=collections.Counter @@ -88,7 +89,7 @@ class BigQueryV2Report(ProfilingSqlReport): ) usage_state_size: Optional[str] = None ingestion_stage: Optional[str] = None - ingestion_stage_durations: Dict[str, str] = field(default_factory=TopKDict) + ingestion_stage_durations: TopKDict[str, float] = field(default_factory=TopKDict) _timer: Optional[PerfTimer] = field( default=None, init=False, repr=False, compare=False @@ -96,7 +97,7 @@ class BigQueryV2Report(ProfilingSqlReport): def set_ingestion_stage(self, project: str, stage: str) -> None: if self._timer: - elapsed = f"{self._timer.elapsed_seconds():.2f}" + elapsed = round(self._timer.elapsed_seconds(), 2) logger.info( f"Time spent in stage <{self.ingestion_stage}>: {elapsed} seconds" ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py index b5c8c74acc089..255a673026252 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py @@ -153,13 +153,18 @@ def make_lineage_edges_from_parsing_result( if upstream_column_info.table == table_urn ) - table_name = str( - BigQueryTableRef.from_bigquery_table( - BigqueryTableIdentifier.from_string_name( - DatasetUrn.create_from_string(table_urn).get_dataset_name() + try: + table_name = str( + BigQueryTableRef.from_bigquery_table( + BigqueryTableIdentifier.from_string_name( + DatasetUrn.create_from_string(table_urn).get_dataset_name() + ) ) ) - ) + except IndexError as e: + logger.debug(f"Unable to parse table urn {table_urn}: {e}") + continue + table_edges[table_name] = LineageEdge( table=table_name, column_mapping=frozenset( diff --git a/metadata-ingestion/src/datahub/utilities/stats_collections.py b/metadata-ingestion/src/datahub/utilities/stats_collections.py index a41139c501175..09a9490abc0fb 100644 --- a/metadata-ingestion/src/datahub/utilities/stats_collections.py +++ b/metadata-ingestion/src/datahub/utilities/stats_collections.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, DefaultDict, Dict, Optional, TypeVar +from typing import Any, Callable, DefaultDict, Dict, Optional, TypeVar, Union from typing_extensions import Protocol @@ -43,7 +43,12 @@ def as_obj(self) -> Dict[_KT, _VT]: ) except TypeError: trimmed_dict = dict(list(self.items())[: self.top_k]) - trimmed_dict[f"... top {self.top_k} of total {len(self)} entries"] = "" # type: ignore + + try: + total_value: Union[_VT, str] = sum(trimmed_dict.values()) # type: ignore + except Exception: + total_value = "" + trimmed_dict[f"... top {self.top_k} of total {len(self)} entries"] = total_value # type: ignore return trimmed_dict From 0def0e5a2e2f7c4a0cd6a674f51339fee2311b62 Mon Sep 17 00:00:00 2001 From: JifeiMei <33741213+JifeiMei@users.noreply.github.com> Date: Wed, 19 Jul 2023 18:38:00 +0800 Subject: [PATCH 157/222] =?UTF-8?q?fix(GreatExpections):=20AssertionRunEve?= =?UTF-8?q?ntClass=20does=20not=20match=20the=20examp=E2=80=A6=20(#8243)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../integrations/great_expectations/action.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/metadata-ingestion/src/datahub/integrations/great_expectations/action.py b/metadata-ingestion/src/datahub/integrations/great_expectations/action.py index 36b85c306a86d..eabf62a4cda2b 100644 --- a/metadata-ingestion/src/datahub/integrations/great_expectations/action.py +++ b/metadata-ingestion/src/datahub/integrations/great_expectations/action.py @@ -317,9 +317,11 @@ def get_assertions_with_results( type=AssertionResultType.SUCCESS if success else AssertionResultType.FAILURE, - rowCount=result.get("element_count"), - missingCount=result.get("missing_count"), - unexpectedCount=result.get("unexpected_count"), + rowCount=parse_int_or_default(result.get("element_count")), + missingCount=parse_int_or_default(result.get("missing_count")), + unexpectedCount=parse_int_or_default( + result.get("unexpected_count") + ), actualAggValue=actualAggValue, externalUrl=docs_link, nativeResults=nativeResults, @@ -699,6 +701,13 @@ def get_platform_instance(self, datasource_name): return None +def parse_int_or_default(value, default_value=None): + if value is None: + return default_value + else: + return int(value) + + def make_dataset_urn_from_sqlalchemy_uri( sqlalchemy_uri, schema_name, From 695de42a8c302fdfeee7fd9d19ef666f5c880819 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 19 Jul 2023 04:16:32 -0700 Subject: [PATCH 158/222] chore(ingest): hide ignore old/new state options (#8438) --- .../datahub/ingestion/source/state/stateful_ingestion_base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py b/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py index 373b10bb9547c..9dd6d27d56ea9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py @@ -68,10 +68,12 @@ class StatefulIngestionConfig(ConfigModel): ignore_old_state: bool = Field( default=False, description="If set to True, ignores the previous checkpoint state.", + hidden_from_docs=True, ) ignore_new_state: bool = Field( default=False, description="If set to True, ignores the current checkpoint state.", + hidden_from_docs=True, ) @pydantic.root_validator() From ee84464406996c75137894ef2b854b2020c0c2a8 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Wed, 19 Jul 2023 06:19:36 -0500 Subject: [PATCH 159/222] docs(env): add env vars authentication (#8436) --- docs/deploy/environment-vars.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/deploy/environment-vars.md b/docs/deploy/environment-vars.md index 0721aaa7964b7..af4ae09c009fd 100644 --- a/docs/deploy/environment-vars.md +++ b/docs/deploy/environment-vars.md @@ -76,3 +76,11 @@ Simply replace the dot, `.`, with an underscore, `_`, and convert to uppercase. | `AWS_GLUE_SCHEMA_REGISTRY_REGION` | `us-east-1` | string | [`GMS`, `MCE Consumer`, `MAE Consumer`] | If using `AWS_GLUE` in the `SCHEMA_REGISTRY_TYPE` variable for the schema registry implementation. | | `AWS_GLUE_SCHEMA_REGISTRY_NAME` | `` | string | [`GMS`, `MCE Consumer`, `MAE Consumer`] | If using `AWS_GLUE` in the `SCHEMA_REGISTRY_TYPE` variable for the schema registry. | | `USE_CONFLUENT_SCHEMA_REGISTRY` | `true` | boolean | [`kafka-setup`] | Enable Confluent schema registry configuration. | + +## Frontend + +| Variable | Default | Unit/Type | Components | Description | +|------------------------------------|----------|-----------|--------------|-------------------------------------------------------------------------------------------------------------------------------------| +| `AUTH_VERBOSE_LOGGING` | `false` | boolean | [`Frontend`] | Enable verbose authentication logging. Enabling this will leak sensisitve information in the logs. Disable when finished debugging. | +| `AUTH_OIDC_GROUPS_CLAIM` | `groups` | string | [`Frontend`] | Claim to use as the user's group. | +| `AUTH_OIDC_EXTRACT_GROUPS_ENABLED` | `false` | boolean | [`Frontend`] | Auto-provision the group from the user's group claim. | From 58125d35ce04738ec94672d6a0f8ba9fdd0b7a24 Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Wed, 19 Jul 2023 13:39:58 -0700 Subject: [PATCH 160/222] =?UTF-8?q?feat(graphql-plugins):=20add=20ability?= =?UTF-8?q?=20for=20plugins=20to=20call=20back=20to=20core=20e=E2=80=A6=20?= =?UTF-8?q?(#8449)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java | 4 +++- .../java/com/linkedin/datahub/graphql/GmsGraphQLPlugin.java | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 1dd07c1873b2f..99d50cdcd6b97 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -340,6 +340,7 @@ import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; import org.dataloader.BatchLoaderContextProvider; @@ -355,6 +356,7 @@ * A {@link GraphQLEngine} configured to provide access to the entities and aspects on the the GMS graph. */ @Slf4j +@Getter public class GmsGraphQLEngine { private final EntityClient entityClient; @@ -613,7 +615,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { * @param builder */ private void configurePluginResolvers(final RuntimeWiring.Builder builder) { - this.graphQLPlugins.forEach(plugin -> plugin.configureExtraResolvers(builder)); + this.graphQLPlugins.forEach(plugin -> plugin.configureExtraResolvers(builder, this)); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLPlugin.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLPlugin.java index ae1e26b610a61..e7ef0c402a1de 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLPlugin.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLPlugin.java @@ -35,9 +35,10 @@ public interface GmsGraphQLPlugin { /** * Optional callback that a plugin can implement to configure any Query, Mutation or Type specific resolvers. - * @param wiringBuilder + * @param wiringBuilder : the builder being used to configure the runtime wiring + * @param baseEngine : a reference to the core engine and its graphql types */ - default void configureExtraResolvers(final RuntimeWiring.Builder wiringBuilder) { + default void configureExtraResolvers(final RuntimeWiring.Builder wiringBuilder, final GmsGraphQLEngine baseEngine) { } From 93fde6bf936dac2a4305354d98735338d2573439 Mon Sep 17 00:00:00 2001 From: RyanHolstien Date: Wed, 19 Jul 2023 20:09:14 -0500 Subject: [PATCH 161/222] feat(io): refactor metadata-io module (#8306) --- .github/scripts/check_event_type.py | 2 +- build.gradle | 1 + datahub-graphql-core/build.gradle | 2 + .../datahub/graphql/GmsGraphQLEngine.java | 10 +- .../datahub/graphql/GmsGraphQLEngineArgs.java | 2 +- .../resolvers/config/AppConfigResolver.java | 2 +- .../CreateTestConnectionRequestResolver.java | 2 +- .../resolvers/mutate/util/LabelUtils.java | 12 +- .../resolvers/mutate/util/OwnerUtils.java | 3 +- .../search/GetQuickFiltersResolver.java | 1 - ...IngestionExecutionRequestResolverTest.java | 2 +- ...eateTestConnectionRequestResolverTest.java | 2 +- datahub-upgrade/build.gradle | 1 + .../upgrade/nocode/DataMigrationStep.java | 2 +- .../datahub/upgrade/nocode/NoCodeUpgrade.java | 3 +- .../elasticsearch/steps/CleanIndicesStep.java | 2 +- ...pgradeCliApplicationTestConfiguration.java | 2 +- docs-website/sidebars.js | 1 + docs/deploy/aws.md | 2 +- docs/dev-guides/timeline.md | 4 +- docs/how/updating-datahub.md | 2 +- docs/plugins.md | 4 +- ingestion-scheduler/build.gradle | 1 + .../ingestion/IngestionScheduler.java | 2 +- .../ingestion/IngestionSchedulerTest.java | 2 +- .../java/com/linkedin/metadata/Constants.java | 3 + metadata-io/build.gradle | 6 +- .../metadata/client/JavaEntityClient.java | 6 +- .../search/custom/QueryConfiguration.java | 108 ------ .../linkedin/metadata/entity/AspectDao.java | 4 +- .../metadata/entity/EntityAspect.java | 2 +- ...ityService.java => EntityServiceImpl.java} | 99 +++--- .../linkedin/metadata/entity/EntityUtils.java | 2 +- .../metadata/graph/SiblingGraphService.java | 2 +- .../graph/elastic/ESGraphQueryDAO.java | 2 +- .../metadata/search/LineageSearchService.java | 2 +- .../AllEntitiesSearchAggregator.java | 2 +- .../search/cache/EntityDocCountCache.java | 2 +- .../indexbuilder/ESIndexBuilder.java | 2 +- .../elasticsearch/query/ESSearchDAO.java | 4 +- .../query/request/CustomizedQueryHandler.java | 2 +- .../query/request/SearchQueryBuilder.java | 77 ++++- .../query/request/SearchRequestHandler.java | 4 +- .../linkedin/metadata/AspectUtilsTest.java | 8 +- .../metadata/ESSampleDataFixture.java | 14 +- .../metadata/ESSearchLineageFixture.java | 10 +- .../entity/AspectMigrationsDaoTest.java | 10 +- .../CassandraAspectMigrationsDaoTest.java | 8 +- .../entity/CassandraEntityServiceTest.java | 18 +- .../entity/DeleteEntityServiceTest.java | 8 +- .../entity/EbeanAspectMigrationsDaoTest.java | 8 +- .../entity/EbeanEntityServiceTest.java | 30 +- .../metadata/entity/EntityServiceTest.java | 232 ++++++------- .../ElasticSearchGraphServiceTest.java | 2 +- .../search/LineageSearchServiceTest.java | 6 +- .../metadata/search/SearchServiceTest.java | 4 +- .../ElasticSearchServiceTest.java | 2 +- .../indexbuilder/ESIndexBuilderTest.java | 2 +- .../elasticsearch/query/ESSearchDAOTest.java | 2 +- .../request/AggregationQueryBuilderTest.java | 2 +- .../request/CustomizedQueryHandlerTest.java | 16 +- .../query/request/SearchQueryBuilderTest.java | 10 +- .../request/SearchRequestHandlerTest.java | 6 +- .../CassandraTimelineServiceTest.java | 6 +- .../timeline/EbeanTimelineServiceTest.java | 6 +- .../timeline/TimelineServiceTest.java | 8 +- .../kafka/MaeConsumerApplicationTest.java | 4 +- ...eConsumerApplicationTestConfiguration.java | 4 +- .../kafka/hook/UpdateIndicesHookTest.java | 4 +- .../spring/MCLSpringTestConfiguration.java | 2 +- .../kafka/MceConsumerApplicationTest.java | 4 +- ...eConsumerApplicationTestConfiguration.java | 2 +- metadata-service/configuration/build.gradle | 14 + .../metadata/config/AssetsConfiguration.java | 0 .../config/AuthPluginConfiguration.java | 0 .../metadata/config/DataHubConfiguration.java | 0 .../metadata/config/EntityProfileConfig.java | 0 .../EntityRegistryPluginConfiguration.java | 0 .../config/IngestionConfiguration.java | 0 .../metadata/config/PluginConfiguration.java | 0 .../metadata/config/PreProcessHooks.java | 0 .../metadata/config/QueriesTabConfig.java | 0 .../config/RetentionPluginConfiguration.java | 0 .../config/SystemUpdateConfiguration.java | 0 .../metadata/config/TestsConfiguration.java | 0 .../metadata/config/ViewsConfiguration.java | 0 .../metadata/config/VisualConfiguration.java | 0 .../config/cache/CacheConfiguration.java | 0 .../EntityDocCountCacheConfiguration.java | 0 .../cache/HomepageCacheConfiguration.java | 0 .../cache/PrimaryCacheConfiguration.java | 0 .../cache/SearchCacheConfiguration.java | 0 .../SearchLineageCacheConfiguration.java | 0 .../config/kafka/KafkaConfiguration.java | 0 .../config/kafka/ListenerConfiguration.java | 0 .../config/kafka/ProducerConfiguration.java | 0 .../kafka/SchemaRegistryConfiguration.java | 0 .../search/BuildIndicesConfiguration.java | 0 .../config/search/CustomConfiguration.java | 2 +- .../search/ElasticSearchConfiguration.java | 2 - .../search/ExactMatchConfiguration.java | 0 .../search/GraphQueryConfiguration.java | 0 .../config/search/PartialConfiguration.java | 0 .../config/search/SearchConfiguration.java | 0 .../search/custom/BoolQueryConfiguration.java | 0 .../custom/CustomSearchConfiguration.java | 0 .../search/custom/QueryConfiguration.java | 35 ++ .../telemetry/TelemetryConfiguration.java | 2 +- .../BatchWriteOperationsOptions.java | 0 .../src/main/resources/application.yml | 0 metadata-service/factories/build.gradle | 1 + .../auth/DataHubTokenServiceFactory.java | 4 +- .../factory/config/ConfigurationProvider.java | 10 +- .../factory/entity/EntityServiceFactory.java | 3 +- .../kafka/DataHubKafkaProducerFactory.java | 2 +- .../kafka/KafkaEventConsumerFactory.java | 2 +- .../kafka/SimpleKafkaConsumerFactory.java | 2 +- .../DUHESchemaRegistryFactory.java | 2 +- .../InternalSchemaRegistryFactory.java | 6 +- .../RecentlyEditedCandidateSourceFactory.java | 4 +- .../search/ElasticSearchServiceFactory.java | 6 +- .../boot/steps/BackfillBrowsePathsV2Step.java | 5 +- .../steps/UpgradeDefaultBrowsePathsStep.java | 4 +- .../IngestDataPlatformInstancesStepTest.java | 6 +- .../openapi/util/MappingUtil.java | 3 +- .../java/entities/EntitiesControllerTest.java | 2 +- .../src/test/java/mock/MockEntityService.java | 7 +- .../resources/entity/AspectResource.java | 3 +- .../entity/BatchIngestionRunResource.java | 2 +- .../resources/entity/EntityResource.java | 3 +- .../resources/entity/AspectResourceTest.java | 6 +- .../registry/SchemaRegistryController.java | 2 +- metadata-service/services/README.md | 5 + metadata-service/services/build.gradle | 73 ++++ .../DataHubUsageEventConstants.java | 0 .../datahubusage/DataHubUsageEventType.java | 0 .../linkedin/metadata/entity/AspectUtils.java | 0 .../metadata/entity/DeleteEntityService.java | 2 +- .../metadata/entity/DeleteEntityUtils.java | 0 .../metadata/entity/EntityService.java | 311 ++++++++++++++++++ .../metadata/entity/IngestProposalResult.java | 12 + .../linkedin/metadata/entity/ListResult.java | 0 .../metadata/entity/RetentionService.java | 2 +- .../metadata/entity/RollbackResult.java | 0 .../metadata/entity/RollbackRunResult.java | 0 .../metadata/entity/UpdateAspectResult.java | 21 ++ .../restoreindices/RestoreIndicesArgs.java | 0 .../restoreindices/RestoreIndicesResult.java | 0 .../retention/BulkApplyRetentionArgs.java | 0 .../retention/BulkApplyRetentionResult.java | 0 .../com/linkedin/metadata/graph/Edge.java | 0 .../linkedin/metadata/graph/GraphClient.java | 0 .../linkedin/metadata/graph/GraphFilters.java | 0 .../metadata/graph/GraphIndexUtils.java | 0 .../linkedin/metadata/graph/GraphService.java | 15 +- .../metadata/graph/RelatedEntitiesResult.java | 0 .../metadata/graph/RelatedEntity.java | 0 .../RecommendationsService.java | 0 .../DomainsCandidateSource.java | 0 .../EntitySearchAggregationSource.java | 0 .../RecentlySearchedSource.java | 0 .../candidatesource/RecommendationSource.java | 0 .../candidatesource/RecommendationUtils.java | 0 .../candidatesource/TopPlatformsSource.java | 0 .../candidatesource/TopTagsSource.java | 0 .../candidatesource/TopTermsSource.java | 0 .../ranker/RecommendationModuleRanker.java | 0 .../ranker/SimpleRecommendationRanker.java | 0 .../registry/SchemaRegistryService.java | 2 +- .../registry/SchemaRegistryServiceImpl.java | 2 +- .../metadata/resource/ResourceReference.java | 0 .../metadata/resource/SubResourceType.java | 0 .../metadata/search/EntitySearchService.java | 4 +- .../metadata/search/utils/QueryUtils.java | 5 +- .../metadata/secret/SecretService.java | 0 .../metadata/service/BaseService.java | 0 .../metadata/service/DataProductService.java | 0 .../metadata/service/DomainService.java | 0 .../metadata/service/GlossaryTermService.java | 0 .../metadata/service/LineageService.java | 0 .../metadata/service/OwnerService.java | 0 .../service/OwnershipTypeService.java | 0 .../metadata/service/QueryService.java | 0 .../metadata/service/SettingsService.java | 0 .../linkedin/metadata/service/TagService.java | 0 .../metadata/service/ViewService.java | 0 .../metadata/shared/ValidationUtils.java | 0 .../systemmetadata/SystemMetadataService.java | 0 .../metadata/timeline/SemanticVersion.java | 0 .../metadata/timeline/TimelineService.java | 0 .../timeline/data/ChangeCategory.java | 0 .../metadata/timeline/data/ChangeEvent.java | 0 .../timeline/data/ChangeOperation.java | 0 .../timeline/data/ChangeTransaction.java | 0 .../timeline/data/PatchOperation.java | 0 .../timeline/data/SemanticChangeType.java | 0 .../timeline/data/SemanticDifference.java | 0 .../timeseries/TimeseriesAspectService.java | 0 .../metadata/service/DomainServiceTest.java | 8 +- .../service/GlossaryTermServiceTest.java | 4 +- .../metadata/service/LineageServiceTest.java | 15 +- .../metadata/service/OwnerServiceTest.java | 4 +- .../service/OwnershipTypeServiceTest.java | 2 +- .../metadata/service/QueryServiceTest.java | 0 .../metadata/service/SettingsServiceTest.java | 0 .../metadata/service/TagServiceTest.java | 4 +- .../metadata/service/ViewServiceTest.java | 4 +- .../gms/servlet/ConfigSearchExport.java | 2 +- settings.gradle | 3 +- 209 files changed, 981 insertions(+), 495 deletions(-) delete mode 100644 metadata-io/src/main/java/com/linkedin/metadata/config/search/custom/QueryConfiguration.java rename metadata-io/src/main/java/com/linkedin/metadata/entity/{EntityService.java => EntityServiceImpl.java} (98%) create mode 100644 metadata-service/configuration/build.gradle rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/AssetsConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/AuthPluginConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/DataHubConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/EntityProfileConfig.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/EntityRegistryPluginConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/IngestionConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/PluginConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/PreProcessHooks.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/QueriesTabConfig.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/RetentionPluginConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/SystemUpdateConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/TestsConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/ViewsConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/VisualConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/cache/CacheConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/cache/EntityDocCountCacheConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/cache/HomepageCacheConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/cache/PrimaryCacheConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/cache/SearchCacheConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/cache/SearchLineageCacheConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/kafka/KafkaConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/kafka/ListenerConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/kafka/ProducerConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/kafka/SchemaRegistryConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/search/BuildIndicesConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/search/CustomConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/search/ElasticSearchConfiguration.java (75%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/search/ExactMatchConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/search/GraphQueryConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/search/PartialConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/search/SearchConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/search/custom/BoolQueryConfiguration.java (100%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/config/search/custom/CustomSearchConfiguration.java (100%) create mode 100644 metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/custom/QueryConfiguration.java rename {metadata-io/src/main/java/com/linkedin/metadata => metadata-service/configuration/src/main/java/com/linkedin/metadata/config}/telemetry/TelemetryConfiguration.java (92%) rename {metadata-io => metadata-service/configuration}/src/main/java/com/linkedin/metadata/timeseries/BatchWriteOperationsOptions.java (100%) rename metadata-service/{factories => configuration}/src/main/resources/application.yml (100%) create mode 100644 metadata-service/services/README.md create mode 100644 metadata-service/services/build.gradle rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventConstants.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/entity/AspectUtils.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/entity/DeleteEntityService.java (99%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/entity/DeleteEntityUtils.java (100%) create mode 100644 metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java create mode 100644 metadata-service/services/src/main/java/com/linkedin/metadata/entity/IngestProposalResult.java rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/entity/ListResult.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/entity/RetentionService.java (99%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/entity/RollbackResult.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/entity/RollbackRunResult.java (100%) create mode 100644 metadata-service/services/src/main/java/com/linkedin/metadata/entity/UpdateAspectResult.java rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/entity/restoreindices/RestoreIndicesArgs.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/entity/restoreindices/RestoreIndicesResult.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/entity/retention/BulkApplyRetentionArgs.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/entity/retention/BulkApplyRetentionResult.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/graph/Edge.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/graph/GraphClient.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/graph/GraphFilters.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/graph/GraphIndexUtils.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/graph/GraphService.java (93%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/graph/RelatedEntitiesResult.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/graph/RelatedEntity.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/recommendation/RecommendationsService.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/recommendation/candidatesource/DomainsCandidateSource.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/recommendation/candidatesource/EntitySearchAggregationSource.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlySearchedSource.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecommendationSource.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecommendationUtils.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/recommendation/candidatesource/TopPlatformsSource.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/recommendation/candidatesource/TopTagsSource.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/recommendation/candidatesource/TopTermsSource.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/recommendation/ranker/RecommendationModuleRanker.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/recommendation/ranker/SimpleRecommendationRanker.java (100%) rename {metadata-io/src/main/java/com/linkedin/metadata/schema => metadata-service/services/src/main/java/com/linkedin/metadata}/registry/SchemaRegistryService.java (89%) rename {metadata-io/src/main/java/com/linkedin/metadata/schema => metadata-service/services/src/main/java/com/linkedin/metadata}/registry/SchemaRegistryServiceImpl.java (98%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/resource/ResourceReference.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/resource/SubResourceType.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/search/EntitySearchService.java (97%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/search/utils/QueryUtils.java (98%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/secret/SecretService.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/service/BaseService.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/service/DataProductService.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/service/DomainService.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/service/GlossaryTermService.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/service/LineageService.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/service/OwnerService.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/service/OwnershipTypeService.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/service/QueryService.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/service/SettingsService.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/service/TagService.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/service/ViewService.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/shared/ValidationUtils.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/systemmetadata/SystemMetadataService.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/timeline/SemanticVersion.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/timeline/TimelineService.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/timeline/data/ChangeCategory.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/timeline/data/ChangeEvent.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/timeline/data/ChangeOperation.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/timeline/data/ChangeTransaction.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/timeline/data/PatchOperation.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/timeline/data/SemanticChangeType.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/timeline/data/SemanticDifference.java (100%) rename {metadata-io => metadata-service/services}/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java (100%) rename {metadata-io => metadata-service/services}/src/test/java/com/linkedin/metadata/service/DomainServiceTest.java (98%) rename {metadata-io => metadata-service/services}/src/test/java/com/linkedin/metadata/service/GlossaryTermServiceTest.java (99%) rename {metadata-io => metadata-service/services}/src/test/java/com/linkedin/metadata/service/LineageServiceTest.java (99%) rename {metadata-io => metadata-service/services}/src/test/java/com/linkedin/metadata/service/OwnerServiceTest.java (98%) rename {metadata-io => metadata-service/services}/src/test/java/com/linkedin/metadata/service/OwnershipTypeServiceTest.java (99%) rename {metadata-io => metadata-service/services}/src/test/java/com/linkedin/metadata/service/QueryServiceTest.java (100%) rename {metadata-io => metadata-service/services}/src/test/java/com/linkedin/metadata/service/SettingsServiceTest.java (100%) rename {metadata-io => metadata-service/services}/src/test/java/com/linkedin/metadata/service/TagServiceTest.java (99%) rename {metadata-io => metadata-service/services}/src/test/java/com/linkedin/metadata/service/ViewServiceTest.java (99%) diff --git a/.github/scripts/check_event_type.py b/.github/scripts/check_event_type.py index f575164a07fc1..c936497a2d307 100644 --- a/.github/scripts/check_event_type.py +++ b/.github/scripts/check_event_type.py @@ -1,7 +1,7 @@ import sys java_events = set() -with open("./metadata-io/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java") as java_file: +with open("./metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java") as java_file: for line in java_file: if '''Event"''' not in line: continue diff --git a/build.gradle b/build.gradle index a13965e057a7c..605b4fcc050e7 100644 --- a/build.gradle +++ b/build.gradle @@ -129,6 +129,7 @@ project.ext.externalDependency = [ 'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1', 'jsonSmart': 'net.minidev:json-smart:2.4.9', 'json': 'org.json:json:20230227', + 'junit': 'junit:junit:4.13.2', 'junitJupiterApi': "org.junit.jupiter:junit-jupiter-api:$junitJupiterVersion", 'junitJupiterParams': "org.junit.jupiter:junit-jupiter-params:$junitJupiterVersion", 'junitJupiterEngine': "org.junit.jupiter:junit-jupiter-engine:$junitJupiterVersion", diff --git a/datahub-graphql-core/build.gradle b/datahub-graphql-core/build.gradle index 12ce7c090c869..8fd45033373dc 100644 --- a/datahub-graphql-core/build.gradle +++ b/datahub-graphql-core/build.gradle @@ -7,6 +7,8 @@ dependencies { compile project(':metadata-service:restli-client') compile project(':metadata-service:auth-impl') compile project(':metadata-service:auth-config') + compile project(':metadata-service:configuration') + compile project(':metadata-service:services') compile project(':metadata-io') compile project(':metadata-utils') diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 99d50cdcd6b97..f22568602d6b4 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -37,7 +37,6 @@ import com.linkedin.datahub.graphql.generated.CorpUser; import com.linkedin.datahub.graphql.generated.CorpUserInfo; import com.linkedin.datahub.graphql.generated.CorpUserViewsSettings; -import com.linkedin.datahub.graphql.generated.OwnershipTypeEntity; import com.linkedin.datahub.graphql.generated.Dashboard; import com.linkedin.datahub.graphql.generated.DashboardInfo; import com.linkedin.datahub.graphql.generated.DashboardStatsSummary; @@ -63,9 +62,9 @@ import com.linkedin.datahub.graphql.generated.InstitutionalMemoryMetadata; import com.linkedin.datahub.graphql.generated.LineageRelationship; import com.linkedin.datahub.graphql.generated.ListAccessTokenResult; -import com.linkedin.datahub.graphql.generated.ListOwnershipTypesResult; import com.linkedin.datahub.graphql.generated.ListDomainsResult; import com.linkedin.datahub.graphql.generated.ListGroupsResult; +import com.linkedin.datahub.graphql.generated.ListOwnershipTypesResult; import com.linkedin.datahub.graphql.generated.ListQueriesResult; import com.linkedin.datahub.graphql.generated.ListTestsResult; import com.linkedin.datahub.graphql.generated.ListViewsResult; @@ -80,6 +79,7 @@ import com.linkedin.datahub.graphql.generated.MLPrimaryKeyProperties; import com.linkedin.datahub.graphql.generated.Notebook; import com.linkedin.datahub.graphql.generated.Owner; +import com.linkedin.datahub.graphql.generated.OwnershipTypeEntity; import com.linkedin.datahub.graphql.generated.PolicyMatchCriterionValue; import com.linkedin.datahub.graphql.generated.QueryEntity; import com.linkedin.datahub.graphql.generated.QuerySubject; @@ -196,9 +196,9 @@ import com.linkedin.datahub.graphql.resolvers.mutate.UpdateParentNodeResolver; import com.linkedin.datahub.graphql.resolvers.mutate.UpdateUserSettingResolver; import com.linkedin.datahub.graphql.resolvers.operation.ReportOperationResolver; +import com.linkedin.datahub.graphql.resolvers.ownership.CreateOwnershipTypeResolver; import com.linkedin.datahub.graphql.resolvers.ownership.DeleteOwnershipTypeResolver; import com.linkedin.datahub.graphql.resolvers.ownership.ListOwnershipTypesResolver; -import com.linkedin.datahub.graphql.resolvers.ownership.CreateOwnershipTypeResolver; import com.linkedin.datahub.graphql.resolvers.ownership.UpdateOwnershipTypeResolver; import com.linkedin.datahub.graphql.resolvers.policy.DeletePolicyResolver; import com.linkedin.datahub.graphql.resolvers.policy.GetGrantedPrivilegesResolver; @@ -303,6 +303,7 @@ import com.linkedin.metadata.config.TestsConfiguration; import com.linkedin.metadata.config.ViewsConfiguration; import com.linkedin.metadata.config.VisualConfiguration; +import com.linkedin.metadata.config.telemetry.TelemetryConfiguration; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.GraphClient; import com.linkedin.metadata.graph.SiblingGraphService; @@ -312,12 +313,11 @@ import com.linkedin.metadata.recommendation.RecommendationsService; import com.linkedin.metadata.secret.SecretService; import com.linkedin.metadata.service.DataProductService; +import com.linkedin.metadata.service.LineageService; import com.linkedin.metadata.service.OwnershipTypeService; import com.linkedin.metadata.service.QueryService; import com.linkedin.metadata.service.SettingsService; import com.linkedin.metadata.service.ViewService; -import com.linkedin.metadata.service.LineageService; -import com.linkedin.metadata.telemetry.TelemetryConfiguration; import com.linkedin.metadata.timeline.TimelineService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; import com.linkedin.metadata.version.GitVersion; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java index 1c4cd09b329d6..cbcf42c4f93d9 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java @@ -16,6 +16,7 @@ import com.linkedin.metadata.config.TestsConfiguration; import com.linkedin.metadata.config.ViewsConfiguration; import com.linkedin.metadata.config.VisualConfiguration; +import com.linkedin.metadata.config.telemetry.TelemetryConfiguration; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.GraphClient; import com.linkedin.metadata.graph.SiblingGraphService; @@ -28,7 +29,6 @@ import com.linkedin.metadata.service.QueryService; import com.linkedin.metadata.service.SettingsService; import com.linkedin.metadata.service.ViewService; -import com.linkedin.metadata.telemetry.TelemetryConfiguration; import com.linkedin.metadata.timeline.TimelineService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; import com.linkedin.metadata.version.GitVersion; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java index 876f3f08c8e25..2c55bc79fe501 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java @@ -26,8 +26,8 @@ import com.linkedin.metadata.config.IngestionConfiguration; import com.linkedin.metadata.config.TestsConfiguration; import com.linkedin.metadata.config.ViewsConfiguration; -import com.linkedin.metadata.telemetry.TelemetryConfiguration; import com.linkedin.metadata.config.VisualConfiguration; +import com.linkedin.metadata.config.telemetry.TelemetryConfiguration; import com.linkedin.metadata.version.GitVersion; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolver.java index 8f4d538ca67ec..1886db62ae450 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolver.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql.resolvers.ingest.execution; +import com.linkedin.metadata.config.IngestionConfiguration; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.StringMap; import com.linkedin.datahub.graphql.QueryContext; @@ -9,7 +10,6 @@ import com.linkedin.entity.client.EntityClient; import com.linkedin.execution.ExecutionRequestInput; import com.linkedin.execution.ExecutionRequestSource; -import com.linkedin.metadata.config.IngestionConfiguration; import com.linkedin.metadata.key.ExecutionRequestKey; import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.IngestionUtils; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LabelUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LabelUtils.java index 1922a02fc1ca0..e2dbf1d3f9c99 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LabelUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LabelUtils.java @@ -135,7 +135,8 @@ public static void addTermsToResource( ) throws URISyntaxException { if (subResource == null || subResource.equals("")) { com.linkedin.common.GlossaryTerms terms = - (com.linkedin.common.GlossaryTerms) getAspectFromEntity(resourceUrn.toString(), GLOSSARY_TERM_ASPECT_NAME, entityService, new GlossaryTerms()); + (com.linkedin.common.GlossaryTerms) getAspectFromEntity(resourceUrn.toString(), GLOSSARY_TERM_ASPECT_NAME, + entityService, new GlossaryTerms()); terms.setAuditStamp(getAuditStamp(actor)); if (!terms.hasTerms()) { @@ -320,7 +321,8 @@ private static MetadataChangeProposal buildRemoveTagsToEntityProposal( EntityService entityService ) { com.linkedin.common.GlobalTags tags = - (com.linkedin.common.GlobalTags) getAspectFromEntity(resource.getResourceUrn(), TAGS_ASPECT_NAME, entityService, new GlobalTags()); + (com.linkedin.common.GlobalTags) getAspectFromEntity(resource.getResourceUrn(), TAGS_ASPECT_NAME, + entityService, new GlobalTags()); if (!tags.hasTags()) { tags.setTags(new TagAssociationArray()); @@ -357,7 +359,8 @@ private static MetadataChangeProposal buildAddTagsToEntityProposal( EntityService entityService ) throws URISyntaxException { com.linkedin.common.GlobalTags tags = - (com.linkedin.common.GlobalTags) getAspectFromEntity(resource.getResourceUrn(), TAGS_ASPECT_NAME, entityService, new GlobalTags()); + (com.linkedin.common.GlobalTags) getAspectFromEntity(resource.getResourceUrn(), TAGS_ASPECT_NAME, + entityService, new GlobalTags()); if (!tags.hasTags()) { tags.setTags(new TagAssociationArray()); @@ -449,7 +452,8 @@ private static MetadataChangeProposal buildAddTermsToEntityProposal( EntityService entityService ) throws URISyntaxException { com.linkedin.common.GlossaryTerms terms = - (com.linkedin.common.GlossaryTerms) getAspectFromEntity(resource.getResourceUrn(), GLOSSARY_TERM_ASPECT_NAME, entityService, new GlossaryTerms()); + (com.linkedin.common.GlossaryTerms) getAspectFromEntity(resource.getResourceUrn(), GLOSSARY_TERM_ASPECT_NAME, + entityService, new GlossaryTerms()); terms.setAuditStamp(getAuditStamp(actor)); if (!terms.hasTerms()) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java index a08419b5226b4..d8a92fb3f6607 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java @@ -72,8 +72,7 @@ public static void removeOwnersFromResources( private static MetadataChangeProposal buildAddOwnersProposal(List owners, Urn resourceUrn, Urn actor, EntityService entityService) { Ownership ownershipAspect = (Ownership) getAspectFromEntity( resourceUrn.toString(), - Constants.OWNERSHIP_ASPECT_NAME, - entityService, + Constants.OWNERSHIP_ASPECT_NAME, entityService, new Ownership()); for (OwnerInput input : owners) { addOwner(ownershipAspect, UrnUtils.getUrn(input.getOwnerUrn()), input.getType(), UrnUtils.getUrn(input.getOwnershipTypeUrn())); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/GetQuickFiltersResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/GetQuickFiltersResolver.java index 782760aca744b..5f4f8dd974328 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/GetQuickFiltersResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/GetQuickFiltersResolver.java @@ -33,7 +33,6 @@ import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.SEARCHABLE_ENTITY_TYPES; import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.resolveView; -import static com.linkedin.datahub.graphql.types.mappers.MapperUtils.*; @Slf4j diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolverTest.java index 35043fa0879f3..7973e49c6efdf 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolverTest.java @@ -1,6 +1,7 @@ package com.linkedin.datahub.graphql.resolvers.ingest.execution; import com.datahub.authentication.Authentication; +import com.linkedin.metadata.config.IngestionConfiguration; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.linkedin.datahub.graphql.QueryContext; @@ -11,7 +12,6 @@ import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; -import com.linkedin.metadata.config.IngestionConfiguration; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetchingEnvironment; diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolverTest.java index 0eb9366f0493b..75df240441965 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolverTest.java @@ -1,10 +1,10 @@ package com.linkedin.datahub.graphql.resolvers.ingest.execution; import com.datahub.authentication.Authentication; +import com.linkedin.metadata.config.IngestionConfiguration; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.CreateTestConnectionRequestInput; import com.linkedin.entity.client.EntityClient; -import com.linkedin.metadata.config.IngestionConfiguration; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetchingEnvironment; import org.mockito.Mockito; diff --git a/datahub-upgrade/build.gradle b/datahub-upgrade/build.gradle index 679e54871cbc8..ad2bf02bfdcc7 100644 --- a/datahub-upgrade/build.gradle +++ b/datahub-upgrade/build.gradle @@ -15,6 +15,7 @@ dependencies { compile project(':metadata-io') compile project(':metadata-service:factories') compile project(':metadata-service:restli-client') + compile project(':metadata-service:configuration') implementation externalDependency.charle compile externalDependency.javaxInject diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/DataMigrationStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/DataMigrationStep.java index 689b1fb997f38..6553bb80bb1fa 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/DataMigrationStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/DataMigrationStep.java @@ -9,11 +9,11 @@ import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.UpgradeStepResult; import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.utils.PegasusUtils; import com.datahub.util.RecordUtils; -import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.ebean.EbeanAspectV1; import com.linkedin.metadata.entity.ebean.EbeanAspectV2; import com.linkedin.metadata.models.EntitySpec; diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java index d1715ce7d66a6..c12ff201faf22 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java @@ -33,8 +33,7 @@ public NoCodeUpgrade( final Authentication systemAuthentication, final RestliEntityClient entityClient) { _steps = buildUpgradeSteps( - server, - entityService, + server, entityService, entityRegistry, systemAuthentication, entityClient); diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/CleanIndicesStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/CleanIndicesStep.java index 99c5e7444d16b..f60aa283c0140 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/CleanIndicesStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/CleanIndicesStep.java @@ -1,11 +1,11 @@ package com.linkedin.datahub.upgrade.system.elasticsearch.steps; +import com.linkedin.metadata.config.search.ElasticSearchConfiguration; import com.linkedin.datahub.upgrade.UpgradeContext; import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.UpgradeStepResult; import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; import com.linkedin.datahub.upgrade.system.elasticsearch.util.IndexUtils; -import com.linkedin.metadata.config.search.ElasticSearchConfiguration; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.shared.ElasticSearchIndexed; import lombok.extern.slf4j.Slf4j; diff --git a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java index 8ae3a832d0aaf..fefc853be8c0b 100644 --- a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java +++ b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java @@ -22,7 +22,7 @@ public class UpgradeCliApplicationTestConfiguration { private EbeanServer ebeanServer; @MockBean - private EntityService entityService; + private EntityService _entityService; @MockBean private SearchService searchService; diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index 85fa61b88ab7e..b5ffd1964d7c1 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -640,6 +640,7 @@ module.exports = { // "metadata-jobs/README", // "docs/how/add-user-data", // "docs/_feature-guide-template" + // - "metadata-service/services/README" // ], ], }; diff --git a/docs/deploy/aws.md b/docs/deploy/aws.md index a8da85c16362d..7b01ffa02a744 100644 --- a/docs/deploy/aws.md +++ b/docs/deploy/aws.md @@ -322,7 +322,7 @@ and [here](../../metadata-service/factories/src/main/java/com/linkedin/gms/facto . A mapping between the property name used in the above two files and the name used in docker/env file can be -found [here](../../metadata-service/factories/src/main/resources/application.yml). +found [here](../../metadata-service/configuration/src/main/resources/application.yml). ### Managed Streaming for Apache Kafka (MSK) diff --git a/docs/dev-guides/timeline.md b/docs/dev-guides/timeline.md index 7f90d17f322ca..966e659b90991 100644 --- a/docs/dev-guides/timeline.md +++ b/docs/dev-guides/timeline.md @@ -18,8 +18,8 @@ For the visually inclined, here is a conceptual diagram that illustrates how to ## Change Event Each modification is modeled as a -[ChangeEvent](../../metadata-io/src/main/java/com/linkedin/metadata/timeline/data/ChangeEvent.java) -which are grouped under [ChangeTransactions](../../metadata-io/src/main/java/com/linkedin/metadata/timeline/data/ChangeTransaction.java) +[ChangeEvent](../../metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/ChangeEvent.java) +which are grouped under [ChangeTransactions](../../metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/ChangeTransaction.java) based on timestamp. A `ChangeEvent` consists of: - `changeType`: An operational type for the change, either `ADD`, `MODIFY`, or `REMOVE` diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 2123b07d166d4..b8ecd689ce381 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -107,7 +107,7 @@ Helm with `--atomic`: In general, it is recommended to not use the `--atomic` se - #6243 apache-ranger authorizer is no longer the core part of DataHub GMS, and it is shifted as plugin. Please refer updated documentation [Configuring Authorization with Apache Ranger](./configuring-authorization-with-apache-ranger.md#configuring-your-datahub-deployment) for configuring `apache-ranger-plugin` in DataHub GMS. - #6243 apache-ranger authorizer as plugin is not supported in DataHub Kubernetes deployment. -- #6243 Authentication and Authorization plugins configuration are removed from [application.yml](../../metadata-service/factories/src/main/resources/application.yml). Refer documentation [Migration Of Plugins From application.yml](../plugins.md#migration-of-plugins-from-applicationyml) for migrating any existing custom plugins. +- #6243 Authentication and Authorization plugins configuration are removed from [application.yml](../../metadata-service/configuration/src/main/resources/application.yml). Refer documentation [Migration Of Plugins From application.yml](../plugins.md#migration-of-plugins-from-applicationyml) for migrating any existing custom plugins. - `datahub check graph-consistency` command has been removed. It was a beta API that we had considered but decided there are better solutions for this. So removing this. - `graphql_url` option of `powerbi-report-server` source deprecated as the options is not used. - #6789 BigQuery ingestion: If `enable_legacy_sharded_table_support` is set to False, sharded table names will be suffixed with \_yyyymmdd to make sure they don't clash with non-sharded tables. This means if stateful ingestion is enabled then old sharded tables will be recreated with a new id and attached tags/glossary terms/etc will need to be added again. _This behavior is not enabled by default yet, but will be enabled by default in a future release._ diff --git a/docs/plugins.md b/docs/plugins.md index 772c877cff646..1d49d99ffc328 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -253,10 +253,10 @@ All other access are forbidden for the plugin. > Disclaimer: In BETA version your plugin can access any port and can read/write to any location on file system, however you should implement the plugin as per above access permission to keep your plugin compatible with upcoming release of DataHub. ## Migration Of Plugins From application.yml -If you have any custom Authentication or Authorization plugin define in `authorization` or `authentication` section of [application.yml](../metadata-service/factories/src/main/resources/application.yml) then migrate them as per below steps. +If you have any custom Authentication or Authorization plugin define in `authorization` or `authentication` section of [application.yml](../metadata-service/configuration/src/main/resources/application.yml) then migrate them as per below steps. 1. Implement Plugin: For Authentication Plugin follow steps of [Implementing an Authentication Plugin](#implementing-an-authentication-plugin) and for Authorization Plugin follow steps of [Implementing an Authorization Plugin](#implementing-an-authorization-plugin) -2. Install Plugin: Install the plugins as per steps mentioned in [Plugin Installation](#plugin-installation). Here you need to map the configuration from [application.yml](../metadata-service/factories/src/main/resources/application.yml) to configuration in `config.yml`. This mapping from `application.yml` to `config.yml` is described below +2. Install Plugin: Install the plugins as per steps mentioned in [Plugin Installation](#plugin-installation). Here you need to map the configuration from [application.yml](../metadata-service/configuration/src/main/resources/application.yml) to configuration in `config.yml`. This mapping from `application.yml` to `config.yml` is described below **Mapping for Authenticators** diff --git a/ingestion-scheduler/build.gradle b/ingestion-scheduler/build.gradle index 3dec8ee400150..b15b5b8c52673 100644 --- a/ingestion-scheduler/build.gradle +++ b/ingestion-scheduler/build.gradle @@ -4,6 +4,7 @@ dependencies { compile project(path: ':metadata-models') compile project(path: ':metadata-io') compile project(path: ':metadata-service:restli-client') + compile project(':metadata-service:configuration') implementation externalDependency.slf4jApi compileOnly externalDependency.lombok annotationProcessor externalDependency.lombok diff --git a/ingestion-scheduler/src/main/java/com/datahub/metadata/ingestion/IngestionScheduler.java b/ingestion-scheduler/src/main/java/com/datahub/metadata/ingestion/IngestionScheduler.java index 5d50c0a326054..e71fe6266b955 100644 --- a/ingestion-scheduler/src/main/java/com/datahub/metadata/ingestion/IngestionScheduler.java +++ b/ingestion-scheduler/src/main/java/com/datahub/metadata/ingestion/IngestionScheduler.java @@ -1,6 +1,7 @@ package com.datahub.metadata.ingestion; import com.datahub.authentication.Authentication; +import com.linkedin.metadata.config.IngestionConfiguration; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; import com.linkedin.common.urn.Urn; @@ -17,7 +18,6 @@ import com.linkedin.ingestion.DataHubIngestionSourceInfo; import com.linkedin.ingestion.DataHubIngestionSourceSchedule; import com.linkedin.metadata.Constants; -import com.linkedin.metadata.config.IngestionConfiguration; import com.linkedin.metadata.key.ExecutionRequestKey; import com.linkedin.metadata.query.ListResult; import com.linkedin.metadata.utils.GenericRecordUtils; diff --git a/ingestion-scheduler/src/test/java/com/datahub/metadata/ingestion/IngestionSchedulerTest.java b/ingestion-scheduler/src/test/java/com/datahub/metadata/ingestion/IngestionSchedulerTest.java index 3397d07d69ceb..51b7fe85f4922 100644 --- a/ingestion-scheduler/src/test/java/com/datahub/metadata/ingestion/IngestionSchedulerTest.java +++ b/ingestion-scheduler/src/test/java/com/datahub/metadata/ingestion/IngestionSchedulerTest.java @@ -1,6 +1,7 @@ package com.datahub.metadata.ingestion; import com.datahub.authentication.Authentication; +import com.linkedin.metadata.config.IngestionConfiguration; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.linkedin.common.UrnArray; @@ -10,7 +11,6 @@ import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.metadata.client.JavaEntityClient; -import com.linkedin.metadata.config.IngestionConfiguration; import com.linkedin.ingestion.DataHubIngestionSourceConfig; import com.linkedin.ingestion.DataHubIngestionSourceInfo; import com.linkedin.ingestion.DataHubIngestionSourceSchedule; diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java index d37ced007133c..972f52b8824ce 100644 --- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -354,6 +354,9 @@ public class Constants { public static final String ELASTICSEARCH_IMPLEMENTATION_OPENSEARCH = "opensearch"; public static final String ELASTICSEARCH_IMPLEMENTATION_ELASTICSEARCH = "elasticsearch"; + // DAO + public static final long LATEST_VERSION = 0; + private Constants() { } } diff --git a/metadata-io/build.gradle b/metadata-io/build.gradle index 88c11c664fe9e..98b741b6f51a6 100644 --- a/metadata-io/build.gradle +++ b/metadata-io/build.gradle @@ -12,15 +12,17 @@ dependencies { compile project(':metadata-events:mxe-avro-1.7') compile project(':metadata-events:mxe-registration') compile project(':metadata-events:mxe-utils-avro-1.7') - compile project(path: ':metadata-models') + compile project(':metadata-models') compile project(':metadata-service:restli-client') + compile project(':metadata-service:configuration') + compile project(':metadata-service:services') compile spec.product.pegasus.data compile spec.product.pegasus.generator compile externalDependency.dgraph4j exclude group: 'com.google.guava', module: 'guava' implementation externalDependency.slf4jApi - testImplementation project(path: ':metadata-integration:java:datahub-client') + testImplementation project(':metadata-integration:java:datahub-client') runtime externalDependency.logbackClassic compileOnly externalDependency.lombok implementation externalDependency.commonsCollections diff --git a/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java b/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java index d541127d74093..911ab993e5789 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java @@ -368,8 +368,7 @@ public SearchResult searchAcrossEntities( @Nullable List facets) throws RemoteInvocationException { final SearchFlags finalFlags = searchFlags != null ? searchFlags : new SearchFlags().setFulltext(true); return ValidationUtils.validateSearchResult( - _searchService.searchAcrossEntities(entities, input, filter, null, start, count, finalFlags, facets), - _entityService); + _searchService.searchAcrossEntities(entities, input, filter, null, start, count, finalFlags, facets), _entityService); } @Nonnull @@ -406,8 +405,7 @@ public LineageSearchResult searchAcrossLineage(@Nonnull Urn sourceUrn, @Nonnull throws RemoteInvocationException { return ValidationUtils.validateLineageSearchResult( _lineageSearchService.searchAcrossLineage(sourceUrn, direction, entities, input, maxHops, filter, - sortCriterion, start, count, startTimeMillis, endTimeMillis, searchFlags), - _entityService); + sortCriterion, start, count, startTimeMillis, endTimeMillis, searchFlags), _entityService); } @Nonnull diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/search/custom/QueryConfiguration.java b/metadata-io/src/main/java/com/linkedin/metadata/config/search/custom/QueryConfiguration.java deleted file mode 100644 index cc252aa3954d3..0000000000000 --- a/metadata-io/src/main/java/com/linkedin/metadata/config/search/custom/QueryConfiguration.java +++ /dev/null @@ -1,108 +0,0 @@ -package com.linkedin.metadata.config.search.custom; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.core.StreamReadConstraints; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; -import lombok.Builder; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.ToString; -import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; -import org.elasticsearch.search.SearchModule; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -import static com.linkedin.metadata.Constants.*; - - -@Slf4j -@Builder(toBuilder = true) -@Getter -@ToString -@EqualsAndHashCode -@JsonDeserialize(builder = QueryConfiguration.QueryConfigurationBuilder.class) -public class QueryConfiguration { - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - static { - OBJECT_MAPPER.setSerializationInclusion(JsonInclude.Include.NON_NULL); - int maxSize = Integer.parseInt(System.getenv().getOrDefault(INGESTION_MAX_SERIALIZED_STRING_LENGTH, MAX_JACKSON_STRING_SIZE)); - OBJECT_MAPPER.getFactory().setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(maxSize).build()); - } - private static final NamedXContentRegistry X_CONTENT_REGISTRY; - static { - SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); - X_CONTENT_REGISTRY = new NamedXContentRegistry(searchModule.getNamedXContents()); - } - - private String queryRegex; - @Builder.Default - private boolean simpleQuery = true; - @Builder.Default - private boolean exactMatchQuery = true; - @Builder.Default - private boolean prefixMatchQuery = true; - private BoolQueryConfiguration boolQuery; - private Map functionScore; - - public FunctionScoreQueryBuilder functionScoreQueryBuilder(QueryBuilder queryBuilder) { - return toFunctionScoreQueryBuilder(queryBuilder, functionScore); - } - - public Optional boolQueryBuilder(String query) { - if (boolQuery != null) { - log.debug("Using custom query configuration queryRegex: {}", queryRegex); - } - return Optional.ofNullable(boolQuery).map(bq -> toBoolQueryBuilder(query, bq)); - } - - @JsonPOJOBuilder(withPrefix = "") - public static class QueryConfigurationBuilder { - } - - private static BoolQueryBuilder toBoolQueryBuilder(String query, BoolQueryConfiguration boolQuery) { - try { - String jsonFragment = OBJECT_MAPPER.writeValueAsString(boolQuery) - .replace("\"{{query_string}}\"", OBJECT_MAPPER.writeValueAsString(query)); - XContentParser parser = XContentType.JSON.xContent().createParser(X_CONTENT_REGISTRY, - LoggingDeprecationHandler.INSTANCE, jsonFragment); - return BoolQueryBuilder.fromXContent(parser); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private static FunctionScoreQueryBuilder toFunctionScoreQueryBuilder(QueryBuilder queryBuilder, - Map params) { - try { - HashMap body = new HashMap<>(params); - if (!body.isEmpty()) { - log.debug("Using custom scoring functions: {}", body); - } - - body.put("query", OBJECT_MAPPER.readValue(queryBuilder.toString(), Map.class)); - - String jsonFragment = OBJECT_MAPPER.writeValueAsString(Map.of( - "function_score", body - )); - XContentParser parser = XContentType.JSON.xContent().createParser(X_CONTENT_REGISTRY, - LoggingDeprecationHandler.INSTANCE, jsonFragment); - return (FunctionScoreQueryBuilder) FunctionScoreQueryBuilder.parseInnerQueryBuilder(parser); - } catch (IOException e) { - throw new RuntimeException(e); - } - } -} diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectDao.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectDao.java index ebb5139d35cae..bf74b1025267f 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectDao.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectDao.java @@ -21,9 +21,9 @@ * Requirements for any implementation: * 1. Being able to map its internal storage representation to {@link EntityAspect}; * 2. Honor the internal versioning semantics. The latest version of any aspect is set to 0 for efficient retrieval. - * In most cases only the latest state of an aspect will be fetched. See {@link EntityService} for more details. + * In most cases only the latest state of an aspect will be fetched. See {@link EntityServiceImpl} for more details. * - * TODO: This interface exposes {@link #runInTransactionWithRetry(Supplier, int)} because {@link EntityService} concerns + * TODO: This interface exposes {@link #runInTransactionWithRetry(Supplier, int)} because {@link EntityServiceImpl} concerns * itself with batching multiple commands into a single transaction. It exposes storage concerns somewhat and it'd be * worth looking into ways to move this responsibility inside {@link AspectDao} implementations. */ diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityAspect.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityAspect.java index ed95e8564d5e9..8296edd615aad 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityAspect.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityAspect.java @@ -10,7 +10,7 @@ import java.sql.Timestamp; /** - * This is an internal representation of an entity aspect record {@link EntityService} and {@link AspectDao} + * This is an internal representation of an entity aspect record {@link EntityServiceImpl} and {@link AspectDao} * implementations are using. While {@link AspectDao} implementations have their own aspect record implementations, * they cary implementation details that should not leak outside. Therefore, this is the type to use in public * {@link AspectDao} methods. diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java similarity index 98% rename from metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java rename to metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java index e5d549b95754d..32c77b66679f9 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java @@ -1,6 +1,7 @@ package com.linkedin.metadata.entity; import com.codahale.metrics.Timer; +import com.linkedin.metadata.config.PreProcessHooks; import com.datahub.util.RecordUtils; import com.datahub.util.exception.ModelConversionException; import com.fasterxml.jackson.core.JsonProcessingException; @@ -43,7 +44,6 @@ import com.linkedin.metadata.Constants; import com.linkedin.metadata.aspect.Aspect; import com.linkedin.metadata.aspect.VersionedAspect; -import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.entity.ebean.EbeanAspectV2; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesResult; @@ -97,7 +97,6 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.persistence.EntityNotFoundException; -import lombok.Value; import lombok.extern.slf4j.Slf4j; import static com.linkedin.metadata.Constants.*; @@ -134,7 +133,7 @@ * TODO: Consider whether we can abstract away virtual versioning semantics to subclasses of this class. */ @Slf4j -public class EntityService { +public class EntityServiceImpl implements EntityService { /** * As described above, the latest version of an aspect should always take the value 0, with @@ -147,25 +146,6 @@ public class EntityService { OBJECT_MAPPER.getFactory().setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(maxSize).build()); } - @Value - public static class UpdateAspectResult { - Urn urn; - RecordTemplate oldValue; - RecordTemplate newValue; - SystemMetadata oldSystemMetadata; - SystemMetadata newSystemMetadata; - MetadataAuditOperation operation; - AuditStamp auditStamp; - long maxVersion; - } - - @Value - public static class IngestProposalResult { - Urn urn; - boolean didUpdate; - boolean queued; - } - private static final int DEFAULT_MAX_TRANSACTION_RETRY = 3; protected final AspectDao _aspectDao; @@ -176,9 +156,6 @@ public static class IngestProposalResult { private final Boolean _alwaysEmitChangeLog; private final UpdateIndicesService _updateIndicesService; private final PreProcessHooks _preProcessHooks; - public static final String DEFAULT_RUN_ID = "no-run-id-provided"; - public static final String BROWSE_PATHS = "browsePaths"; - public static final String DATA_PLATFORM_INSTANCE = "dataPlatformInstance"; protected static final int MAX_KEYS_PER_QUERY = 500; private static final int URN_NUM_BYTES_LIMIT = 512; @@ -186,7 +163,7 @@ public static class IngestProposalResult { // TODO(iprentic): Move this to a common utils location once used in other places private static final String DELIMITER_SEPARATOR = "␟"; - public EntityService( + public EntityServiceImpl( @Nonnull final AspectDao aspectDao, @Nonnull final EventProducer producer, @Nonnull final EntityRegistry entityRegistry, @@ -249,6 +226,7 @@ public Map> getLatestAspects( } @Nonnull + @Override public Map getLatestAspectsForUrn(@Nonnull final Urn urn, @Nonnull final Set aspectNames) { Map batchGetResults = getLatestAspect(new HashSet<>(Arrays.asList(urn)), aspectNames); @@ -273,6 +251,7 @@ public Map getLatestAspectsForUrn(@Nonnull final Urn urn * @return the {@link RecordTemplate} representation of the requested aspect object, or null if one cannot be found */ @Nullable + @Override public RecordTemplate getAspect(@Nonnull final Urn urn, @Nonnull final String aspectName, @Nonnull long version) { log.debug("Invoked getAspect with urn: {}, aspectName: {}, version: {}", urn, aspectName, version); @@ -294,6 +273,7 @@ public RecordTemplate getAspect(@Nonnull final Urn urn, @Nonnull final String as * @return a map of {@link Urn} to {@link Entity} object */ @Nullable + @Override public EntityResponse getEntityV2( @Nonnull final String entityName, @Nonnull final Urn urn, @@ -310,6 +290,7 @@ public EntityResponse getEntityV2( * @param aspectNames set of aspects to fetch * @return a map of {@link Urn} to {@link Entity} object */ + @Override public Map getEntitiesV2( @Nonnull final String entityName, @Nonnull final Set urns, @@ -328,6 +309,7 @@ public Map getEntitiesV2( * @param aspectNames set of aspects to fetch * @return a map of {@link Urn} to {@link Entity} object */ + @Override public Map getEntitiesVersionedV2( @Nonnull final Set versionedUrns, @Nonnull final Set aspectNames) throws URISyntaxException { @@ -345,6 +327,7 @@ public Map getEntitiesVersionedV2( * @param aspectNames set of aspects to fetch * @return a map of {@link Urn} to {@link EnvelopedAspect} object */ + @Override public Map> getLatestEnvelopedAspects( // TODO: entityName is unused, can we remove this as a param? @Nonnull String entityName, @@ -368,6 +351,7 @@ public Map> getLatestEnvelopedAspects( * @param aspectNames set of aspects to fetch * @return a map of {@link Urn} to {@link EnvelopedAspect} object */ + @Override public Map> getVersionedEnvelopedAspects( @Nonnull Set versionedUrns, @Nonnull Set aspectNames) throws URISyntaxException { @@ -434,6 +418,7 @@ private Map> getCorrespondingAspects(Set listLatestAspects( @Nonnull final String entityName, @Nonnull final String aspectName, @@ -719,6 +707,7 @@ void validateUrn(@Nonnull final Urn urn) { } } + @Override public void ingestAspects(@Nonnull final Urn urn, @Nonnull List> aspectRecordsToIngest, @Nonnull final AuditStamp auditStamp, @Nullable SystemMetadata systemMetadata) { @@ -746,6 +735,7 @@ public void ingestAspects(@Nonnull final Urn urn, @Nonnull List logger) { RestoreIndicesResult result = new RestoreIndicesResult(); int ignored = 0; @@ -1217,6 +1212,7 @@ public RestoreIndicesResult restoreIndices(@Nonnull RestoreIndicesArgs args, @No * successful update * @return the {@link RecordTemplate} representation of the requested aspect object */ + @Override public RecordTemplate updateAspect( @Nonnull final Urn urn, @Nonnull final String entityName, @@ -1240,6 +1236,7 @@ public RecordTemplate updateAspect( * @param start the start offset * @param count the count */ + @Override public ListUrnsResult listUrns(@Nonnull final String entityName, final int start, final int count) { log.debug("Invoked listUrns with entityName: {}, start: {}, count: {}", entityName, start, count); @@ -1269,6 +1266,7 @@ public ListUrnsResult listUrns(@Nonnull final String entityName, final int start /** * Default implementations. Subclasses should feel free to override if it's more efficient to do so. */ + @Override public Entity getEntity(@Nonnull final Urn urn, @Nonnull final Set aspectNames) { return getEntities(Collections.singleton(urn), aspectNames).values().stream().findFirst().orElse(null); } @@ -1283,6 +1281,7 @@ public Entity getEntity(@Nonnull final Urn urn, @Nonnull final Set aspec * @return a map of {@link Urn} to {@link Entity} object */ @Deprecated + @Override public Map getEntities(@Nonnull final Set urns, @Nonnull Set aspectNames) { log.debug("Invoked getEntities with urns {}, aspects {}", urns, aspectNames); if (urns.isEmpty()) { @@ -1293,6 +1292,7 @@ public Map getEntities(@Nonnull final Set urns, @Nonnull Set toEntity(entry.getValue()))); } + @Override public void produceMetadataAuditEvent(@Nonnull final Urn urn, @Nonnull final String aspectName, @Nullable final RecordTemplate oldAspectValue, @Nullable final RecordTemplate newAspectValue, @Nullable final SystemMetadata oldSystemMetadata, @Nullable final SystemMetadata newSystemMetadata, @@ -1316,6 +1316,7 @@ protected Snapshot buildKeySnapshot(@Nonnull final Urn urn) { return toSnapshotUnion(toSnapshotRecord(urn, ImmutableList.of(toAspectUnion(urn, keyAspectValue)))); } + @Override public void produceMetadataAuditEventForKey(@Nonnull final Urn urn, @Nullable final SystemMetadata newSystemMetadata) { @@ -1332,11 +1333,13 @@ public void produceMetadataAuditEventForKey(@Nonnull final Urn urn, * @param aspectSpec AspectSpec of the aspect being updated * @param metadataChangeLog metadata change log to push into MCL kafka topic */ + @Override public void produceMetadataChangeLog(@Nonnull final Urn urn, AspectSpec aspectSpec, @Nonnull final MetadataChangeLog metadataChangeLog) { _producer.produceMetadataChangeLog(urn, aspectSpec, metadataChangeLog); } + @Override public void produceMetadataChangeLog(@Nonnull final Urn urn, @Nonnull String entityName, @Nonnull String aspectName, @Nonnull final AspectSpec aspectSpec, @Nullable final RecordTemplate oldAspectValue, @Nullable final RecordTemplate newAspectValue, @Nullable final SystemMetadata oldSystemMetadata, @@ -1346,11 +1349,13 @@ public void produceMetadataChangeLog(@Nonnull final Urn urn, @Nonnull String ent produceMetadataChangeLog(urn, aspectSpec, metadataChangeLog); } + @Override public RecordTemplate getLatestAspect(@Nonnull final Urn urn, @Nonnull final String aspectName) { log.debug("Invoked getLatestAspect with urn {}, aspect {}", urn, aspectName); return getAspect(urn, aspectName, ASPECT_LATEST_VERSION); } + @Override public void ingestEntities(@Nonnull final List entities, @Nonnull final AuditStamp auditStamp, @Nonnull final List systemMetadata) { log.debug("Invoked ingestEntities with entities {}, audit stamp {}", entities, auditStamp); @@ -1358,6 +1363,7 @@ public void ingestEntities(@Nonnull final List entities, @Nonnull final .forEach(pair -> ingestEntity(pair.getFirst(), auditStamp, pair.getSecond())); } + @Override public void ingestEntity(Entity entity, AuditStamp auditStamp) { SystemMetadata generatedSystemMetadata = new SystemMetadata(); generatedSystemMetadata.setRunId(DEFAULT_RUN_ID); @@ -1366,6 +1372,7 @@ public void ingestEntity(Entity entity, AuditStamp auditStamp) { ingestEntity(entity, auditStamp, generatedSystemMetadata); } + @Override public void ingestEntity(@Nonnull Entity entity, @Nonnull AuditStamp auditStamp, @Nonnull SystemMetadata systemMetadata) { log.debug("Invoked ingestEntity with entity {}, audit stamp {} systemMetadata {}", entity, auditStamp, systemMetadata.toString()); @@ -1408,15 +1415,16 @@ private boolean isAspectMissing(String entityType, String aspectName, Set> generateDefaultAspectsIfMissing(@Nonnull final Urn urn, Set includedAspects) { Set aspectsToGet = new HashSet<>(); String entityType = urnToEntityName(urn); - boolean shouldCheckBrowsePath = isAspectMissing(entityType, BROWSE_PATHS, includedAspects); + boolean shouldCheckBrowsePath = isAspectMissing(entityType, BROWSE_PATHS_ASPECT_NAME, includedAspects); if (shouldCheckBrowsePath) { - aspectsToGet.add(BROWSE_PATHS); + aspectsToGet.add(BROWSE_PATHS_ASPECT_NAME); } boolean shouldCheckBrowsePathV2 = isAspectMissing(entityType, BROWSE_PATHS_V2_ASPECT_NAME, includedAspects); @@ -1424,9 +1432,9 @@ public List> generateDefaultAspectsIfMissing(@Nonnu aspectsToGet.add(BROWSE_PATHS_V2_ASPECT_NAME); } - boolean shouldCheckDataPlatform = isAspectMissing(entityType, DATA_PLATFORM_INSTANCE, includedAspects); + boolean shouldCheckDataPlatform = isAspectMissing(entityType, DATA_PLATFORM_INSTANCE_ASPECT_NAME, includedAspects); if (shouldCheckDataPlatform) { - aspectsToGet.add(DATA_PLATFORM_INSTANCE); + aspectsToGet.add(DATA_PLATFORM_INSTANCE_ASPECT_NAME); } List> aspects = new ArrayList<>(); @@ -1441,10 +1449,10 @@ public List> generateDefaultAspectsIfMissing(@Nonnu aspects.add(Pair.of(keyAspectName, keyAspect)); } - if (shouldCheckBrowsePath && latestAspects.get(BROWSE_PATHS) == null) { + if (shouldCheckBrowsePath && latestAspects.get(BROWSE_PATHS_ASPECT_NAME) == null) { try { BrowsePaths generatedBrowsePath = buildDefaultBrowsePath(urn); - aspects.add(Pair.of(BROWSE_PATHS, generatedBrowsePath)); + aspects.add(Pair.of(BROWSE_PATHS_ASPECT_NAME, generatedBrowsePath)); } catch (URISyntaxException e) { log.error("Failed to parse urn: {}", urn); } @@ -1459,9 +1467,9 @@ public List> generateDefaultAspectsIfMissing(@Nonnu } } - if (shouldCheckDataPlatform && latestAspects.get(DATA_PLATFORM_INSTANCE) == null) { + if (shouldCheckDataPlatform && latestAspects.get(DATA_PLATFORM_INSTANCE_ASPECT_NAME) == null) { DataPlatformInstanceUtils.buildDataPlatformInstance(entityType, keyAspect) - .ifPresent(aspect -> aspects.add(Pair.of(DATA_PLATFORM_INSTANCE, aspect))); + .ifPresent(aspect -> aspects.add(Pair.of(DATA_PLATFORM_INSTANCE_ASPECT_NAME, aspect))); } return aspects; @@ -1481,6 +1489,7 @@ private void ingestSnapshotUnion(@Nonnull final Snapshot snapshotUnion, @Nonnull ingestAspects(urn, aspectRecordsToIngest, auditStamp, systemMetadata); } + @Override public Snapshot buildSnapshot(@Nonnull final Urn urn, @Nonnull final RecordTemplate aspectValue) { // if the aspect value is the key, we do not need to include the key a second time if (PegasusUtils.getAspectNameFromSchema(aspectValue.schema()).equals(getKeyAspectName(urn))) { @@ -1499,20 +1508,24 @@ protected RecordTemplate buildKeyAspect(@Nonnull final Urn urn) { return EntityKeyUtils.convertUrnToEntityKey(urn, keySpec); } + @Override public AspectSpec getKeyAspectSpec(@Nonnull final Urn urn) { return getKeyAspectSpec(urnToEntityName(urn)); } + @Override public AspectSpec getKeyAspectSpec(@Nonnull final String entityName) { final EntitySpec spec = _entityRegistry.getEntitySpec(entityName); return spec.getKeyAspectSpec(); } + @Override public Optional getAspectSpec(@Nonnull final String entityName, @Nonnull final String aspectName) { final EntitySpec entitySpec = _entityRegistry.getEntitySpec(entityName); return Optional.ofNullable(entitySpec.getAspectSpec(aspectName)); } + @Override public String getKeyAspectName(@Nonnull final Urn urn) { final EntitySpec spec = _entityRegistry.getEntitySpec(urnToEntityName(urn)); final AspectSpec keySpec = spec.getKeyAspectSpec(); @@ -1577,10 +1590,12 @@ private Map> buildEntityToValidAspects(final EntityRegistry entry -> entry.getAspectSpecs().stream().map(AspectSpec::getName).collect(Collectors.toSet()))); } + @Override public EntityRegistry getEntityRegistry() { return _entityRegistry; } + @Override public void setRetentionService(RetentionService retentionService) { _retentionService = retentionService; } @@ -1589,19 +1604,23 @@ protected Set getEntityAspectNames(final Urn entityUrn) { return getEntityAspectNames(urnToEntityName(entityUrn)); } + @Override public Set getEntityAspectNames(final String entityName) { return _entityToValidAspects.get(entityName); } + @Override public void setWritable(boolean canWrite) { log.debug("Setting writable to {}", canWrite); _aspectDao.setWritable(canWrite); } + @Override public RollbackRunResult rollbackRun(List aspectRows, String runId, boolean hardDelete) { return rollbackWithConditions(aspectRows, Collections.singletonMap("runId", runId), hardDelete); } + @Override public RollbackRunResult rollbackWithConditions(List aspectRows, Map conditions, boolean hardDelete) { List removedAspects = new ArrayList<>(); AtomicInteger rowsDeletedFromEntityDeletion = new AtomicInteger(0); @@ -1630,6 +1649,7 @@ public RollbackRunResult rollbackWithConditions(List aspectRow return new RollbackRunResult(removedAspects, rowsDeletedFromEntityDeletion.get()); } + @Override public RollbackRunResult deleteUrn(Urn urn) { List removedAspects = new ArrayList<>(); Integer rowsDeletedFromEntityDeletion = 0; @@ -1677,6 +1697,7 @@ public RollbackRunResult deleteUrn(Urn urn) { * @param urn the urn of the entity to check * @return true if the entity exists, false otherwise */ + @Override public Boolean exists(Urn urn) { final Set aspectsToFetch = getEntityAspectNames(urn); final List dbKeys = aspectsToFetch.stream() @@ -1693,6 +1714,7 @@ public Boolean exists(Urn urn) { * @param urn the urn to check * @return true is the entity is soft deleted, false otherwise. */ + @Override public Boolean isSoftDeleted(@Nonnull final Urn urn) { Objects.requireNonNull(urn, "urn is required"); final RecordTemplate statusAspect = getLatestAspect(urn, STATUS_ASPECT_NAME); @@ -1700,6 +1722,7 @@ public Boolean isSoftDeleted(@Nonnull final Urn urn) { } @Nullable + @Override public RollbackResult deleteAspect(String urn, String aspectName, @Nonnull Map conditions, boolean hardDelete) { // Validate pre-conditions before running queries Urn entityUrn; @@ -2062,6 +2085,7 @@ private RecordTemplate updateAspect( * This method currently supports datasets, charts, dashboards, data flows, data jobs, and glossary terms. */ @Nonnull + @Override public BrowsePaths buildDefaultBrowsePath(final @Nonnull Urn urn) throws URISyntaxException { Character dataPlatformDelimiter = getDataPlatformDelimiter(urn); String defaultBrowsePath = getDefaultBrowsePath(urn, this.getEntityRegistry(), dataPlatformDelimiter); @@ -2079,6 +2103,7 @@ public BrowsePaths buildDefaultBrowsePath(final @Nonnull Urn urn) throws URISynt * will have a basic "Default" folder added to their browsePathV2. */ @Nonnull + @Override public BrowsePathsV2 buildDefaultBrowsePathV2(final @Nonnull Urn urn, boolean useContainerPaths) throws URISyntaxException { Character dataPlatformDelimiter = getDataPlatformDelimiter(urn); return BrowsePathV2Utils.getDefaultBrowsePathV2(urn, this.getEntityRegistry(), dataPlatformDelimiter, this, useContainerPaths); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java index 53099e900ee4d..abc444faa04a2 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java @@ -16,7 +16,7 @@ import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; -import static com.linkedin.metadata.entity.EntityService.*; +import static com.linkedin.metadata.Constants.*; @Slf4j diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/SiblingGraphService.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/SiblingGraphService.java index f3ba33a5f4081..7a2f0825b31cc 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/SiblingGraphService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/SiblingGraphService.java @@ -43,7 +43,7 @@ public EntityLineageResult getLineage(@Nonnull Urn entityUrn, @Nonnull LineageDi new HashSet<>(), null, null), - _entityService); + _entityService); } /** diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java index d5812f15eccac..8df7a9600ca94 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java @@ -1,6 +1,7 @@ package com.linkedin.metadata.graph.elastic; import com.codahale.metrics.Timer; +import com.linkedin.metadata.config.search.GraphQueryConfiguration; import com.datahub.util.exception.ESQueryException; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; @@ -9,7 +10,6 @@ import com.linkedin.common.UrnArrayArray; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; -import com.linkedin.metadata.config.search.GraphQueryConfiguration; import com.linkedin.metadata.graph.GraphFilters; import com.linkedin.metadata.graph.LineageDirection; import com.linkedin.metadata.graph.LineageRelationship; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java index c7cdd604968a5..5fb1ab0889e71 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.search; +import com.linkedin.metadata.config.cache.SearchLineageCacheConfiguration; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -9,7 +10,6 @@ import com.linkedin.data.template.LongMap; import com.linkedin.data.template.StringArray; import com.linkedin.metadata.Constants; -import com.linkedin.metadata.config.cache.SearchLineageCacheConfiguration; import com.linkedin.metadata.graph.EntityLineageResult; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.graph.LineageDirection; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/aggregator/AllEntitiesSearchAggregator.java b/metadata-io/src/main/java/com/linkedin/metadata/search/aggregator/AllEntitiesSearchAggregator.java index 8b9c712d8d7a3..ee93edaf2480c 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/aggregator/AllEntitiesSearchAggregator.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/aggregator/AllEntitiesSearchAggregator.java @@ -1,9 +1,9 @@ package com.linkedin.metadata.search.aggregator; import com.codahale.metrics.Timer; +import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; import com.linkedin.data.template.GetMode; import com.linkedin.data.template.LongMap; -import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.query.SearchFlags; import com.linkedin.metadata.query.filter.Filter; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/cache/EntityDocCountCache.java b/metadata-io/src/main/java/com/linkedin/metadata/search/cache/EntityDocCountCache.java index d76b137131156..95f208e185df1 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/cache/EntityDocCountCache.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/cache/EntityDocCountCache.java @@ -1,7 +1,7 @@ package com.linkedin.metadata.search.cache; -import com.google.common.base.Suppliers; import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; +import com.google.common.base.Suppliers; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.utils.ConcurrencyUtils; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java index 9affa4c05a566..14f67ddcbf337 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.search.elasticsearch.indexbuilder; +import com.linkedin.metadata.config.search.ElasticSearchConfiguration; import com.google.common.collect.ImmutableMap; import com.linkedin.metadata.search.utils.ESUtils; @@ -21,7 +22,6 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; -import com.linkedin.metadata.config.search.ElasticSearchConfiguration; import com.linkedin.util.Pair; import io.github.resilience4j.retry.Retry; import io.github.resilience4j.retry.RetryConfig; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java index 89874b08eef9d..51e08763cd7c8 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java @@ -1,12 +1,12 @@ package com.linkedin.metadata.search.elasticsearch.query; import com.codahale.metrics.Timer; +import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.datahub.util.exception.ESQueryException; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.annotations.VisibleForTesting; import com.linkedin.data.template.LongMap; -import com.linkedin.metadata.config.search.SearchConfiguration; -import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.query.AutoCompleteResult; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/CustomizedQueryHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/CustomizedQueryHandler.java index 6186a90114880..55a3474fd9f35 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/CustomizedQueryHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/CustomizedQueryHandler.java @@ -1,7 +1,7 @@ package com.linkedin.metadata.search.elasticsearch.query.request; -import com.linkedin.metadata.config.search.custom.QueryConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; +import com.linkedin.metadata.config.search.custom.QueryConfiguration; import lombok.Builder; import lombok.Getter; import lombok.extern.slf4j.Slf4j; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java index 1f54d3bbbf2ad..397352358d05f 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java @@ -3,14 +3,22 @@ import com.linkedin.metadata.config.search.ExactMatchConfiguration; import com.linkedin.metadata.config.search.PartialConfiguration; import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.config.search.custom.BoolQueryConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.config.search.custom.QueryConfiguration; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkedin.metadata.Constants; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.SearchableFieldSpec; import com.linkedin.metadata.models.annotation.SearchScoreAnnotation; import com.linkedin.metadata.models.annotation.SearchableAnnotation; +import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -22,9 +30,15 @@ import javax.annotation.Nullable; import com.linkedin.metadata.search.utils.ESUtils; +import lombok.extern.slf4j.Slf4j; import org.elasticsearch.common.lucene.search.function.CombineFunction; import org.elasticsearch.common.lucene.search.function.FieldValueFactorFunction; import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilder; @@ -34,11 +48,23 @@ import org.elasticsearch.index.query.functionscore.FieldValueFactorFunctionBuilder; import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; +import org.elasticsearch.search.SearchModule; import static com.linkedin.metadata.models.SearchableFieldSpecExtractor.PRIMARY_URN_SEARCH_PROPERTIES; - +@Slf4j public class SearchQueryBuilder { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + static { + OBJECT_MAPPER.setSerializationInclusion(JsonInclude.Include.NON_NULL); + int maxSize = Integer.parseInt(System.getenv().getOrDefault(Constants.INGESTION_MAX_SERIALIZED_STRING_LENGTH, Constants.MAX_JACKSON_STRING_SIZE)); + OBJECT_MAPPER.getFactory().setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(maxSize).build()); + } + private static final NamedXContentRegistry X_CONTENT_REGISTRY; + static { + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); + X_CONTENT_REGISTRY = new NamedXContentRegistry(searchModule.getNamedXContents()); + } public static final String STRUCTURED_QUERY_PREFIX = "\\\\/q "; private final ExactMatchConfiguration exactMatchConfiguration; @@ -72,7 +98,7 @@ private QueryBuilder buildInternalQuery(@Nullable QueryConfiguration customQuery @Nonnull String query, boolean fulltext) { final String sanitizedQuery = query.replaceFirst("^:+", ""); final BoolQueryBuilder finalQuery = Optional.ofNullable(customQueryConfig) - .flatMap(cqc -> cqc.boolQueryBuilder(sanitizedQuery)) + .flatMap(cqc -> boolQueryBuilder(cqc, sanitizedQuery)) .orElse(QueryBuilders.boolQuery()); if (fulltext && !query.startsWith(STRUCTURED_QUERY_PREFIX)) { @@ -229,7 +255,7 @@ private FunctionScoreQueryBuilder buildScoreFunctions(@Nullable QueryConfigurati if (customQueryConfig != null) { // Prefer configuration function scoring over annotation scoring - return customQueryConfig.functionScoreQueryBuilder(queryBuilder); + return functionScoreQueryBuilder(customQueryConfig, queryBuilder); } else { return QueryBuilders.functionScoreQuery(queryBuilder, buildAnnotationScoreFunctions(entitySpecs)) .scoreMode(FunctionScoreQuery.ScoreMode.AVG) // Average score functions @@ -297,4 +323,49 @@ private static FieldValueFactorFunction.Modifier mapModifier(SearchScoreAnnotati return FieldValueFactorFunction.Modifier.NONE; } } + + public FunctionScoreQueryBuilder functionScoreQueryBuilder(QueryConfiguration customQueryConfiguration, + QueryBuilder queryBuilder) { + return toFunctionScoreQueryBuilder(queryBuilder, customQueryConfiguration.getFunctionScore()); + } + + public Optional boolQueryBuilder(QueryConfiguration customQueryConfiguration, String query) { + if (customQueryConfiguration.getBoolQuery() != null) { + log.debug("Using custom query configuration queryRegex: {}", customQueryConfiguration.getQueryRegex()); + } + return Optional.ofNullable(customQueryConfiguration.getBoolQuery()).map(bq -> toBoolQueryBuilder(query, bq)); + } + + private BoolQueryBuilder toBoolQueryBuilder(String query, BoolQueryConfiguration boolQuery) { + try { + String jsonFragment = OBJECT_MAPPER.writeValueAsString(boolQuery) + .replace("\"{{query_string}}\"", OBJECT_MAPPER.writeValueAsString(query)); + XContentParser parser = XContentType.JSON.xContent().createParser(X_CONTENT_REGISTRY, + LoggingDeprecationHandler.INSTANCE, jsonFragment); + return BoolQueryBuilder.fromXContent(parser); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private FunctionScoreQueryBuilder toFunctionScoreQueryBuilder(QueryBuilder queryBuilder, + Map params) { + try { + HashMap body = new HashMap<>(params); + if (!body.isEmpty()) { + log.debug("Using custom scoring functions: {}", body); + } + + body.put("query", OBJECT_MAPPER.readValue(queryBuilder.toString(), Map.class)); + + String jsonFragment = OBJECT_MAPPER.writeValueAsString(Map.of( + "function_score", body + )); + XContentParser parser = XContentType.JSON.xContent().createParser(X_CONTENT_REGISTRY, + LoggingDeprecationHandler.INSTANCE, jsonFragment); + return (FunctionScoreQueryBuilder) FunctionScoreQueryBuilder.parseInnerQueryBuilder(parser); + } catch (IOException e) { + throw new RuntimeException(e); + } + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java index 21cec705d57f3..58d855b6f73ad 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java @@ -1,13 +1,13 @@ package com.linkedin.metadata.search.elasticsearch.query.request; +import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.DoubleMap; import com.linkedin.data.template.LongMap; -import com.linkedin.metadata.config.search.SearchConfiguration; -import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.SearchableFieldSpec; import com.linkedin.metadata.models.annotation.SearchableAnnotation; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/AspectUtilsTest.java b/metadata-io/src/test/java/com/linkedin/metadata/AspectUtilsTest.java index 69579352b8acf..36ebec5a42849 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/AspectUtilsTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/AspectUtilsTest.java @@ -1,13 +1,13 @@ package com.linkedin.metadata; +import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.common.FabricType; import com.linkedin.common.urn.DataPlatformUrn; import com.linkedin.common.urn.DatasetUrn; import com.linkedin.dataset.DatasetProperties; import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.entity.AspectUtils; -import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.EntityServiceImpl; import com.linkedin.metadata.entity.TestEntityRegistry; import com.linkedin.metadata.entity.ebean.EbeanAspectDao; import com.linkedin.metadata.event.EventProducer; @@ -45,7 +45,7 @@ public void testAdditionalChanges() { EventProducer mockProducer = mock(EventProducer.class); PreProcessHooks preProcessHooks = new PreProcessHooks(); preProcessHooks.setUiEnabled(true); - EntityService entityService = new EntityService(aspectDao, mockProducer, _testEntityRegistry, true, + EntityServiceImpl entityServiceImpl = new EntityServiceImpl(aspectDao, mockProducer, _testEntityRegistry, true, null, preProcessHooks); MetadataChangeProposal proposal1 = new MetadataChangeProposal(); @@ -56,7 +56,7 @@ public void testAdditionalChanges() { proposal1.setEntityType("dataset"); proposal1.setChangeType(ChangeType.PATCH); - List proposalList = AspectUtils.getAdditionalChanges(proposal1, entityService); + List proposalList = AspectUtils.getAdditionalChanges(proposal1, entityServiceImpl); // proposals for key aspect, browsePath, browsePathV2, dataPlatformInstance Assert.assertEquals(proposalList.size(), 4); Assert.assertEquals(proposalList.get(0).getChangeType(), ChangeType.UPSERT); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/ESSampleDataFixture.java b/metadata-io/src/test/java/com/linkedin/metadata/ESSampleDataFixture.java index a1393bbd65a47..64237ff30d2a4 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/ESSampleDataFixture.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/ESSampleDataFixture.java @@ -1,18 +1,18 @@ package com.linkedin.metadata; -import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; -import com.linkedin.entity.client.EntityClient; -import com.linkedin.metadata.client.JavaEntityClient; -import com.linkedin.metadata.config.search.CustomConfiguration; import com.linkedin.metadata.config.PreProcessHooks; +import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; +import com.linkedin.metadata.config.search.CustomConfiguration; import com.linkedin.metadata.config.search.ElasticSearchConfiguration; import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; +import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.client.JavaEntityClient; import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.entity.EntityAspect; import com.linkedin.metadata.entity.EntityAspectIdentifier; -import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; -import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.EntityServiceImpl; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.aggregator.AllEntitiesSearchAggregator; @@ -191,7 +191,7 @@ protected EntityClient entityClient( PreProcessHooks preProcessHooks = new PreProcessHooks(); preProcessHooks.setUiEnabled(true); return new JavaEntityClient( - new EntityService(mockAspectDao, null, entityRegistry, true, null, + new EntityServiceImpl(mockAspectDao, null, entityRegistry, true, null, preProcessHooks), null, entitySearchService, diff --git a/metadata-io/src/test/java/com/linkedin/metadata/ESSearchLineageFixture.java b/metadata-io/src/test/java/com/linkedin/metadata/ESSearchLineageFixture.java index 89ae579a9db01..9cedda9bfb035 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/ESSearchLineageFixture.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/ESSearchLineageFixture.java @@ -1,15 +1,15 @@ package com.linkedin.metadata; -import com.linkedin.entity.client.EntityClient; -import com.linkedin.metadata.client.JavaEntityClient; import com.linkedin.metadata.config.PreProcessHooks; +import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; import com.linkedin.metadata.config.cache.SearchLineageCacheConfiguration; import com.linkedin.metadata.config.search.ElasticSearchConfiguration; -import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; import com.linkedin.metadata.config.search.GraphQueryConfiguration; import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; -import com.linkedin.metadata.entity.EntityService; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.client.JavaEntityClient; +import com.linkedin.metadata.entity.EntityServiceImpl; import com.linkedin.metadata.graph.elastic.ESGraphQueryDAO; import com.linkedin.metadata.graph.elastic.ESGraphWriteDAO; import com.linkedin.metadata.graph.elastic.ElasticSearchGraphService; @@ -226,7 +226,7 @@ protected EntityClient entityClient( PreProcessHooks preProcessHooks = new PreProcessHooks(); preProcessHooks.setUiEnabled(true); return new JavaEntityClient( - new EntityService(null, null, entityRegistry, true, null, + new EntityServiceImpl(null, null, entityRegistry, true, null, preProcessHooks), null, entitySearchService, diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/AspectMigrationsDaoTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/AspectMigrationsDaoTest.java index 9304080460fce..6a331647583d2 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/AspectMigrationsDaoTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/AspectMigrationsDaoTest.java @@ -31,7 +31,7 @@ abstract public class AspectMigrationsDaoTest { protected final EntityRegistry _testEntityRegistry; protected EventProducer _mockProducer; - protected EntityService _entityService; + protected EntityServiceImpl _entityServiceImpl; protected RetentionService _retentionService; protected UpdateIndicesService _mockUpdateIndicesService; @@ -46,7 +46,7 @@ public void testListAllUrns() throws AssertionError { final int totalAspects = 30; final int pageSize = 25; final int lastPageSize = 5; - Map ingestedAspects = AspectIngestionUtils.ingestCorpUserKeyAspects(_entityService, totalAspects); + Map ingestedAspects = AspectIngestionUtils.ingestCorpUserKeyAspects(_entityServiceImpl, totalAspects); List ingestedUrns = ingestedAspects.keySet().stream().map(Urn::toString).collect(Collectors.toList()); List seenUrns = new ArrayList<>(); @@ -78,8 +78,8 @@ public void testListAllUrns() throws AssertionError { @Test public void testCountEntities() throws AssertionError { - AspectIngestionUtils.ingestCorpUserInfoAspects(_entityService, 11); - AspectIngestionUtils.ingestChartInfoAspects(_entityService, 22); + AspectIngestionUtils.ingestCorpUserInfoAspects(_entityServiceImpl, 11); + AspectIngestionUtils.ingestChartInfoAspects(_entityServiceImpl, 22); final int expected = 33; long actual = _migrationsDao.countEntities(); @@ -92,7 +92,7 @@ public void testCheckIfAspectExists() throws AssertionError { boolean actual = _migrationsDao.checkIfAspectExists(CORP_USER_INFO_ASPECT_NAME); assertFalse(actual); - AspectIngestionUtils.ingestCorpUserInfoAspects(_entityService, 1); + AspectIngestionUtils.ingestCorpUserInfoAspects(_entityServiceImpl, 1); actual = _migrationsDao.checkIfAspectExists(CORP_USER_INFO_ASPECT_NAME); assertTrue(actual); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraAspectMigrationsDaoTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraAspectMigrationsDaoTest.java index 7e80a67e304c3..70161fe640707 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraAspectMigrationsDaoTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraAspectMigrationsDaoTest.java @@ -1,8 +1,8 @@ package com.linkedin.metadata.entity; +import com.linkedin.metadata.config.PreProcessHooks; import com.datastax.oss.driver.api.core.CqlSession; import com.linkedin.metadata.CassandraTestUtils; -import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.entity.cassandra.CassandraAspectDao; import com.linkedin.metadata.entity.cassandra.CassandraRetentionService; import com.linkedin.metadata.event.EventProducer; @@ -49,10 +49,10 @@ private void configureComponents() { _mockUpdateIndicesService = mock(UpdateIndicesService.class); PreProcessHooks preProcessHooks = new PreProcessHooks(); preProcessHooks.setUiEnabled(true); - _entityService = new EntityService(dao, _mockProducer, _testEntityRegistry, true, _mockUpdateIndicesService, + _entityServiceImpl = new EntityServiceImpl(dao, _mockProducer, _testEntityRegistry, true, _mockUpdateIndicesService, preProcessHooks); - _retentionService = new CassandraRetentionService(_entityService, session, 1000); - _entityService.setRetentionService(_retentionService); + _retentionService = new CassandraRetentionService(_entityServiceImpl, session, 1000); + _entityServiceImpl.setRetentionService(_retentionService); _migrationsDao = dao; } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraEntityServiceTest.java index 6e062fd14fc21..50e562b76c4e6 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraEntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraEntityServiceTest.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.entity; +import com.linkedin.metadata.config.PreProcessHooks; import com.datastax.oss.driver.api.core.CqlSession; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.RecordTemplate; @@ -7,7 +8,6 @@ import com.linkedin.metadata.AspectGenerationUtils; import com.linkedin.metadata.AspectIngestionUtils; import com.linkedin.metadata.CassandraTestUtils; -import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.entity.cassandra.CassandraAspectDao; import com.linkedin.metadata.entity.cassandra.CassandraRetentionService; import com.linkedin.metadata.event.EventProducer; @@ -35,7 +35,7 @@ * A class that knows how to configure {@link EntityServiceTest} to run integration tests against a Cassandra database. * * This class also contains all the test methods where realities of an underlying storage leak into the - * {@link EntityService} in the form of subtle behavior differences. Ideally that should never happen, and it'd be + * {@link EntityServiceImpl} in the form of subtle behavior differences. Ideally that should never happen, and it'd be * great to address captured differences. */ public class CassandraEntityServiceTest extends EntityServiceTest { @@ -69,10 +69,10 @@ private void configureComponents() { _mockUpdateIndicesService = mock(UpdateIndicesService.class); PreProcessHooks preProcessHooks = new PreProcessHooks(); preProcessHooks.setUiEnabled(true); - _entityService = new EntityService(_aspectDao, _mockProducer, _testEntityRegistry, true, + _entityServiceImpl = new EntityServiceImpl(_aspectDao, _mockProducer, _testEntityRegistry, true, _mockUpdateIndicesService, preProcessHooks); - _retentionService = new CassandraRetentionService(_entityService, session, 1000); - _entityService.setRetentionService(_retentionService); + _retentionService = new CassandraRetentionService(_entityServiceImpl, session, 1000); + _entityServiceImpl.setRetentionService(_retentionService); } /** @@ -99,7 +99,7 @@ public void testIngestListLatestAspects() throws AssertionError { final int expectedTotalPages = 4; final int expectedEntitiesInLastPage = 10; - Map writtenAspects = AspectIngestionUtils.ingestCorpUserInfoAspects(_entityService, totalEntities); + Map writtenAspects = AspectIngestionUtils.ingestCorpUserInfoAspects(_entityServiceImpl, totalEntities); Set writtenUrns = writtenAspects.keySet(); String entity = writtenUrns.stream().findFirst().get().getEntityType(); String aspect = AspectGenerationUtils.getAspectName(new CorpUserInfo()); @@ -111,7 +111,7 @@ public void testIngestListLatestAspects() throws AssertionError { int expectedEntityCount = isLastPage ? expectedEntitiesInLastPage : pageSize; int expectedNextStart = isLastPage ? -1 : pageStart + pageSize; - ListResult page = _entityService.listLatestAspects(entity, aspect, pageStart, pageSize); + ListResult page = _entityServiceImpl.listLatestAspects(entity, aspect, pageStart, pageSize); // Check paging metadata works as expected assertEquals(page.getNextStart(), expectedNextStart); @@ -147,7 +147,7 @@ public void testIngestListUrns() throws AssertionError { final int expectedTotalPages = 4; final int expectedEntitiesInLastPage = 10; - Map writtenAspects = AspectIngestionUtils.ingestCorpUserKeyAspects(_entityService, totalEntities); + Map writtenAspects = AspectIngestionUtils.ingestCorpUserKeyAspects(_entityServiceImpl, totalEntities); Set writtenUrns = writtenAspects.keySet(); String entity = writtenUrns.stream().findFirst().get().getEntityType(); @@ -157,7 +157,7 @@ public void testIngestListUrns() throws AssertionError { int pageStart = pageNo * pageSize; int expectedEntityCount = isLastPage ? expectedEntitiesInLastPage : pageSize; - ListUrnsResult page = _entityService.listUrns(entity, pageStart, pageSize); + ListUrnsResult page = _entityServiceImpl.listUrns(entity, pageStart, pageSize); // Check paging metadata works as expected assertEquals(page.getStart().intValue(), pageStart); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java index 759680abe7b4d..98f9ce241b850 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityServiceTest.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.entity; +import com.linkedin.metadata.config.PreProcessHooks; import com.datahub.util.RecordUtils; import com.google.common.collect.ImmutableList; import com.linkedin.common.AuditStamp; @@ -9,7 +10,6 @@ import com.linkedin.entity.EntityResponse; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; -import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.entity.ebean.EbeanAspectDao; import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.graph.GraphService; @@ -37,7 +37,7 @@ public class DeleteEntityServiceTest { protected EbeanAspectDao _aspectDao; - protected EntityService _entityService; + protected EntityServiceImpl _entityServiceImpl; protected GraphService _graphService = Mockito.mock(GraphService.class); protected DeleteEntityService _deleteEntityService; @@ -52,9 +52,9 @@ public DeleteEntityServiceTest() { _mockUpdateIndicesService = mock(UpdateIndicesService.class); PreProcessHooks preProcessHooks = new PreProcessHooks(); preProcessHooks.setUiEnabled(true); - _entityService = new EntityService(_aspectDao, mock(EventProducer.class), _entityRegistry, true, + _entityServiceImpl = new EntityServiceImpl(_aspectDao, mock(EventProducer.class), _entityRegistry, true, _mockUpdateIndicesService, preProcessHooks); - _deleteEntityService = new DeleteEntityService(_entityService, _graphService); + _deleteEntityService = new DeleteEntityService(_entityServiceImpl, _graphService); } /** diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanAspectMigrationsDaoTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanAspectMigrationsDaoTest.java index 2aa66ae7d9d3b..62f8827b574b8 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanAspectMigrationsDaoTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanAspectMigrationsDaoTest.java @@ -1,7 +1,7 @@ package com.linkedin.metadata.entity; -import com.linkedin.metadata.EbeanTestUtils; import com.linkedin.metadata.config.PreProcessHooks; +import com.linkedin.metadata.EbeanTestUtils; import com.linkedin.metadata.entity.ebean.EbeanAspectDao; import com.linkedin.metadata.entity.ebean.EbeanRetentionService; import com.linkedin.metadata.event.EventProducer; @@ -29,10 +29,10 @@ public void setupTest() { _mockUpdateIndicesService = mock(UpdateIndicesService.class); PreProcessHooks preProcessHooks = new PreProcessHooks(); preProcessHooks.setUiEnabled(true); - _entityService = new EntityService(dao, _mockProducer, _testEntityRegistry, true, + _entityServiceImpl = new EntityServiceImpl(dao, _mockProducer, _testEntityRegistry, true, _mockUpdateIndicesService, preProcessHooks); - _retentionService = new EbeanRetentionService(_entityService, server, 1000); - _entityService.setRetentionService(_retentionService); + _retentionService = new EbeanRetentionService(_entityServiceImpl, server, 1000); + _entityServiceImpl.setRetentionService(_retentionService); _migrationsDao = dao; } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java index 3d5b8c6cfaa2b..9126aad62895d 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.entity; +import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.DataTemplateUtil; @@ -7,7 +8,6 @@ import com.linkedin.identity.CorpUserInfo; import com.linkedin.metadata.AspectGenerationUtils; import com.linkedin.metadata.EbeanTestUtils; -import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.entity.ebean.EbeanAspectDao; import com.linkedin.metadata.entity.ebean.EbeanRetentionService; import com.linkedin.metadata.event.EventProducer; @@ -34,7 +34,7 @@ * A class that knows how to configure {@link EntityServiceTest} to run integration tests against a relational database. * * This class also contains all the test methods where realities of an underlying storage leak into the - * {@link EntityService} in the form of subtle behavior differences. Ideally that should never happen, and it'd be + * {@link EntityServiceImpl} in the form of subtle behavior differences. Ideally that should never happen, and it'd be * great to address captured differences. */ public class EbeanEntityServiceTest extends EntityServiceTest { @@ -51,10 +51,10 @@ public void setupTest() { _mockUpdateIndicesService = mock(UpdateIndicesService.class); PreProcessHooks preProcessHooks = new PreProcessHooks(); preProcessHooks.setUiEnabled(true); - _entityService = new EntityService(_aspectDao, _mockProducer, _testEntityRegistry, true, + _entityServiceImpl = new EntityServiceImpl(_aspectDao, _mockProducer, _testEntityRegistry, true, _mockUpdateIndicesService, preProcessHooks); - _retentionService = new EbeanRetentionService(_entityService, server, 1000); - _entityService.setRetentionService(_retentionService); + _retentionService = new EbeanRetentionService(_entityServiceImpl, server, 1000); + _entityServiceImpl.setRetentionService(_retentionService); } /** @@ -86,18 +86,18 @@ public void testIngestListLatestAspects() throws AssertionError { // Ingest CorpUserInfo Aspect #1 CorpUserInfo writeAspect1 = AspectGenerationUtils.createCorpUserInfo("email@test.com"); - _entityService.ingestAspect(entityUrn1, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspect(entityUrn1, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #2 CorpUserInfo writeAspect2 = AspectGenerationUtils.createCorpUserInfo("email2@test.com"); - _entityService.ingestAspect(entityUrn2, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspect(entityUrn2, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #3 CorpUserInfo writeAspect3 = AspectGenerationUtils.createCorpUserInfo("email3@test.com"); - _entityService.ingestAspect(entityUrn3, aspectName, writeAspect3, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspect(entityUrn3, aspectName, writeAspect3, TEST_AUDIT_STAMP, metadata1); // List aspects - ListResult batch1 = _entityService.listLatestAspects(entityUrn1.getEntityType(), aspectName, 0, 2); + ListResult batch1 = _entityServiceImpl.listLatestAspects(entityUrn1.getEntityType(), aspectName, 0, 2); assertEquals(batch1.getNextStart(), 2); assertEquals(batch1.getPageSize(), 2); @@ -107,7 +107,7 @@ public void testIngestListLatestAspects() throws AssertionError { assertTrue(DataTemplateUtil.areEqual(writeAspect1, batch1.getValues().get(0))); assertTrue(DataTemplateUtil.areEqual(writeAspect2, batch1.getValues().get(1))); - ListResult batch2 = _entityService.listLatestAspects(entityUrn1.getEntityType(), aspectName, 2, 2); + ListResult batch2 = _entityServiceImpl.listLatestAspects(entityUrn1.getEntityType(), aspectName, 2, 2); assertEquals(batch2.getValues().size(), 1); assertTrue(DataTemplateUtil.areEqual(writeAspect3, batch2.getValues().get(0))); } @@ -131,18 +131,18 @@ public void testIngestListUrns() throws AssertionError { // Ingest CorpUserInfo Aspect #1 RecordTemplate writeAspect1 = AspectGenerationUtils.createCorpUserKey(entityUrn1); - _entityService.ingestAspect(entityUrn1, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspect(entityUrn1, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #2 RecordTemplate writeAspect2 = AspectGenerationUtils.createCorpUserKey(entityUrn2); - _entityService.ingestAspect(entityUrn2, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspect(entityUrn2, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #3 RecordTemplate writeAspect3 = AspectGenerationUtils.createCorpUserKey(entityUrn3); - _entityService.ingestAspect(entityUrn3, aspectName, writeAspect3, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspect(entityUrn3, aspectName, writeAspect3, TEST_AUDIT_STAMP, metadata1); // List aspects urns - ListUrnsResult batch1 = _entityService.listUrns(entityUrn1.getEntityType(), 0, 2); + ListUrnsResult batch1 = _entityServiceImpl.listUrns(entityUrn1.getEntityType(), 0, 2); assertEquals(batch1.getStart().intValue(), 0); assertEquals(batch1.getCount().intValue(), 2); @@ -151,7 +151,7 @@ public void testIngestListUrns() throws AssertionError { assertEquals(entityUrn1.toString(), batch1.getEntities().get(0).toString()); assertEquals(entityUrn2.toString(), batch1.getEntities().get(1).toString()); - ListUrnsResult batch2 = _entityService.listUrns(entityUrn1.getEntityType(), 2, 2); + ListUrnsResult batch2 = _entityServiceImpl.listUrns(entityUrn1.getEntityType(), 2, 2); assertEquals(batch2.getStart().intValue(), 2); assertEquals(batch2.getCount().intValue(), 1); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java index 1d11bd786c4cf..d485981f32a07 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java @@ -72,7 +72,7 @@ import static org.testng.Assert.*; /** - * A class to test {@link EntityService} + * A class to test {@link EntityServiceImpl} * * This class is generic to allow same integration tests to be reused to test all supported storage backends. * If you're adding another storage backend - you should create a new test class that extends this one providing @@ -87,7 +87,7 @@ */ abstract public class EntityServiceTest { - protected EntityService _entityService; + protected EntityServiceImpl _entityServiceImpl; protected T_AD _aspectDao; protected T_RS _retentionService; @@ -124,10 +124,10 @@ public void testIngestGetEntity() throws Exception { SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(); // 1. Ingest Entity - _entityService.ingestEntity(writeEntity, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestEntity(writeEntity, TEST_AUDIT_STAMP, metadata1); // 2. Retrieve Entity - com.linkedin.entity.Entity readEntity = _entityService.getEntity(entityUrn, Collections.emptySet()); + com.linkedin.entity.Entity readEntity = _entityServiceImpl.getEntity(entityUrn, Collections.emptySet()); // 3. Compare Entity Objects assertEquals(readEntity.getValue().getCorpUserSnapshot().getAspects().size(), 2); // Key + Info aspect. @@ -161,10 +161,10 @@ public void testAddKey() throws Exception { SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(); // 1. Ingest Entity - _entityService.ingestEntity(writeEntity, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestEntity(writeEntity, TEST_AUDIT_STAMP, metadata1); // 2. Retrieve Entity - com.linkedin.entity.Entity readEntity = _entityService.getEntity(entityUrn, Collections.emptySet()); + com.linkedin.entity.Entity readEntity = _entityServiceImpl.getEntity(entityUrn, Collections.emptySet()); // 3. Compare Entity Objects assertEquals(readEntity.getValue().getCorpUserSnapshot().getAspects().size(), 2); // Key + Info aspect. @@ -202,12 +202,12 @@ public void testIngestGetEntities() throws Exception { SystemMetadata metadata2 = AspectGenerationUtils.createSystemMetadata(1625792690, "run-123"); // 1. Ingest Entities - _entityService.ingestEntities(ImmutableList.of(writeEntity1, writeEntity2), TEST_AUDIT_STAMP, + _entityServiceImpl.ingestEntities(ImmutableList.of(writeEntity1, writeEntity2), TEST_AUDIT_STAMP, ImmutableList.of(metadata1, metadata2)); // 2. Retrieve Entities Map readEntities = - _entityService.getEntities(ImmutableSet.of(entityUrn1, entityUrn2), Collections.emptySet()); + _entityServiceImpl.getEntities(ImmutableSet.of(entityUrn1, entityUrn2), Collections.emptySet()); // 3. Compare Entity Objects @@ -279,12 +279,12 @@ public void testIngestGetEntitiesV2() throws Exception { String keyName = "corpUserKey"; // 1. Ingest Entities - _entityService.ingestEntities(ImmutableList.of(writeEntity1, writeEntity2), TEST_AUDIT_STAMP, + _entityServiceImpl.ingestEntities(ImmutableList.of(writeEntity1, writeEntity2), TEST_AUDIT_STAMP, ImmutableList.of(metadata1, metadata2)); // 2. Retrieve Entities Map readEntities = - _entityService.getEntitiesV2("corpuser", ImmutableSet.of(entityUrn1, entityUrn2), ImmutableSet.of(aspectName)); + _entityServiceImpl.getEntitiesV2("corpuser", ImmutableSet.of(entityUrn1, entityUrn2), ImmutableSet.of(aspectName)); // 3. Compare Entity Objects @@ -347,12 +347,12 @@ public void testIngestGetEntitiesVersionedV2() throws Exception { String keyName = "corpUserKey"; // 1. Ingest Entities - _entityService.ingestEntities(ImmutableList.of(writeEntity1, writeEntity2), TEST_AUDIT_STAMP, + _entityServiceImpl.ingestEntities(ImmutableList.of(writeEntity1, writeEntity2), TEST_AUDIT_STAMP, ImmutableList.of(metadata1, metadata2)); // 2. Retrieve Entities Map readEntities = - _entityService.getEntitiesVersionedV2(ImmutableSet.of(versionedUrn1, versionedUrn2), ImmutableSet.of(aspectName)); + _entityServiceImpl.getEntitiesVersionedV2(ImmutableSet.of(versionedUrn1, versionedUrn2), ImmutableSet.of(aspectName)); // 3. Compare Entity Objects @@ -414,9 +414,9 @@ public void testIngestAspectsGetLatestAspects() throws Exception { SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(); - _entityService.ingestAspects(entityUrn, pairToIngest, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspects(entityUrn, pairToIngest, TEST_AUDIT_STAMP, metadata1); - Map latestAspects = _entityService.getLatestAspectsForUrn( + Map latestAspects = _entityServiceImpl.getLatestAspectsForUrn( entityUrn, new HashSet<>(Arrays.asList(aspectName1, aspectName2)) ); @@ -445,7 +445,7 @@ public void testReingestAspectsGetLatestAspects() throws Exception { SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(); - _entityService.ingestAspects(entityUrn, pairToIngest, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspects(entityUrn, pairToIngest, TEST_AUDIT_STAMP, metadata1); final MetadataChangeLog initialChangeLog = new MetadataChangeLog(); initialChangeLog.setEntityType(entityUrn.getEntityType()); @@ -470,7 +470,7 @@ public void testReingestAspectsGetLatestAspects() throws Exception { restateChangeLog.setPreviousAspectValue(aspect); restateChangeLog.setPreviousSystemMetadata(simulatePullFromDB(metadata1, SystemMetadata.class)); - Map latestAspects = _entityService.getLatestAspectsForUrn( + Map latestAspects = _entityServiceImpl.getLatestAspectsForUrn( entityUrn, new HashSet<>(List.of(aspectName1)) ); @@ -484,7 +484,7 @@ public void testReingestAspectsGetLatestAspects() throws Exception { // Mockito detects the previous invocation and throws an error in verifying the second call unless invocations are cleared clearInvocations(_mockProducer); - _entityService.ingestAspects(entityUrn, pairToIngest, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspects(entityUrn, pairToIngest, TEST_AUDIT_STAMP, metadata1); verify(_mockProducer, times(1)).produceMetadataChangeLog(Mockito.eq(entityUrn), Mockito.any(), Mockito.eq(restateChangeLog)); @@ -508,7 +508,7 @@ public void testReingestLineageAspect() throws Exception { SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(); - _entityService.ingestAspects(entityUrn, pairToIngest, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspects(entityUrn, pairToIngest, TEST_AUDIT_STAMP, metadata1); final MetadataChangeLog initialChangeLog = new MetadataChangeLog(); initialChangeLog.setEntityType(entityUrn.getEntityType()); @@ -533,7 +533,7 @@ public void testReingestLineageAspect() throws Exception { restateChangeLog.setPreviousAspectValue(aspect); restateChangeLog.setPreviousSystemMetadata(simulatePullFromDB(metadata1, SystemMetadata.class)); - Map latestAspects = _entityService.getLatestAspectsForUrn( + Map latestAspects = _entityServiceImpl.getLatestAspectsForUrn( entityUrn, new HashSet<>(List.of(aspectName1)) ); @@ -547,7 +547,7 @@ public void testReingestLineageAspect() throws Exception { // Mockito detects the previous invocation and throws an error in verifying the second call unless invocations are cleared clearInvocations(_mockProducer); - _entityService.ingestAspects(entityUrn, pairToIngest, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspects(entityUrn, pairToIngest, TEST_AUDIT_STAMP, metadata1); verify(_mockProducer, times(1)).produceMetadataChangeLog(Mockito.eq(entityUrn), Mockito.any(), Mockito.eq(restateChangeLog)); @@ -578,7 +578,7 @@ public void testReingestLineageProposal() throws Exception { mcp1.setSystemMetadata(metadata1); mcp1.setAspectName(UPSTREAM_LINEAGE_ASPECT_NAME); - _entityService.ingestProposal(mcp1, TEST_AUDIT_STAMP, false); + _entityServiceImpl.ingestProposal(mcp1, TEST_AUDIT_STAMP, false); final MetadataChangeLog initialChangeLog = new MetadataChangeLog(); initialChangeLog.setEntityType(entityUrn.getEntityType()); @@ -601,7 +601,7 @@ public void testReingestLineageProposal() throws Exception { restateChangeLog.setPreviousAspectValue(genericAspect); restateChangeLog.setPreviousSystemMetadata(simulatePullFromDB(metadata1, SystemMetadata.class)); - Map latestAspects = _entityService.getLatestAspectsForUrn( + Map latestAspects = _entityServiceImpl.getLatestAspectsForUrn( entityUrn, new HashSet<>(List.of(aspectName1)) ); @@ -613,7 +613,7 @@ public void testReingestLineageProposal() throws Exception { // Mockito detects the previous invocation and throws an error in verifying the second call unless invocations are cleared clearInvocations(_mockProducer); - _entityService.ingestProposal(mcp1, TEST_AUDIT_STAMP, false); + _entityServiceImpl.ingestProposal(mcp1, TEST_AUDIT_STAMP, false); verify(_mockProducer, times(1)).produceMetadataChangeLog(Mockito.eq(entityUrn), Mockito.any(), Mockito.eq(restateChangeLog)); @@ -640,7 +640,7 @@ public void testIngestTimeseriesAspect() throws Exception { genericAspect.setValue(ByteString.unsafeWrap(datasetProfileSerialized)); genericAspect.setContentType("application/json"); gmce.setAspect(genericAspect); - _entityService.ingestProposal(gmce, TEST_AUDIT_STAMP, false); + _entityServiceImpl.ingestProposal(gmce, TEST_AUDIT_STAMP, false); } @Test @@ -659,7 +659,7 @@ public void testAsyncProposalVersioned() throws Exception { genericAspect.setValue(ByteString.unsafeWrap(datasetPropertiesSerialized)); genericAspect.setContentType("application/json"); gmce.setAspect(genericAspect); - _entityService.ingestProposal(gmce, TEST_AUDIT_STAMP, true); + _entityServiceImpl.ingestProposal(gmce, TEST_AUDIT_STAMP, true); verify(_mockProducer, times(0)).produceMetadataChangeLog(Mockito.eq(entityUrn), Mockito.any(), Mockito.any()); verify(_mockProducer, times(1)).produceMetadataChangeProposal(Mockito.eq(entityUrn), @@ -685,7 +685,7 @@ public void testAsyncProposalTimeseries() throws Exception { genericAspect.setValue(ByteString.unsafeWrap(datasetProfileSerialized)); genericAspect.setContentType("application/json"); gmce.setAspect(genericAspect); - _entityService.ingestProposal(gmce, TEST_AUDIT_STAMP, true); + _entityServiceImpl.ingestProposal(gmce, TEST_AUDIT_STAMP, true); verify(_mockProducer, times(1)).produceMetadataChangeLog(Mockito.eq(entityUrn), Mockito.any(), Mockito.any()); verify(_mockProducer, times(0)).produceMetadataChangeProposal(Mockito.eq(entityUrn), @@ -704,9 +704,9 @@ public void testUpdateGetAspect() throws AssertionError { CorpUserInfo writeAspect = AspectGenerationUtils.createCorpUserInfo("email@test.com"); // Validate retrieval of CorpUserInfo Aspect #1 - _entityService.updateAspect(entityUrn, "corpuser", aspectName, corpUserInfoSpec, writeAspect, TEST_AUDIT_STAMP, 1, + _entityServiceImpl.updateAspect(entityUrn, "corpuser", aspectName, corpUserInfoSpec, writeAspect, TEST_AUDIT_STAMP, 1, true); - RecordTemplate readAspect1 = _entityService.getAspect(entityUrn, aspectName, 1); + RecordTemplate readAspect1 = _entityServiceImpl.getAspect(entityUrn, aspectName, 1); assertTrue(DataTemplateUtil.areEqual(writeAspect, readAspect1)); verify(_mockProducer, times(1)).produceMetadataChangeLog(Mockito.eq(entityUrn), Mockito.eq(corpUserInfoSpec), Mockito.any()); @@ -715,9 +715,9 @@ public void testUpdateGetAspect() throws AssertionError { writeAspect.setEmail("newemail@test.com"); // Validate retrieval of CorpUserInfo Aspect #2 - _entityService.updateAspect(entityUrn, "corpuser", aspectName, corpUserInfoSpec, writeAspect, TEST_AUDIT_STAMP, 1, + _entityServiceImpl.updateAspect(entityUrn, "corpuser", aspectName, corpUserInfoSpec, writeAspect, TEST_AUDIT_STAMP, 1, false); - RecordTemplate readAspect2 = _entityService.getAspect(entityUrn, aspectName, 1); + RecordTemplate readAspect2 = _entityServiceImpl.getAspect(entityUrn, aspectName, 1); assertTrue(DataTemplateUtil.areEqual(writeAspect, readAspect2)); verifyNoMoreInteractions(_mockProducer); } @@ -734,22 +734,22 @@ public void testGetAspectAtVersion() throws AssertionError { CorpUserInfo writeAspect = AspectGenerationUtils.createCorpUserInfo("email@test.com"); // Validate retrieval of CorpUserInfo Aspect #1 - _entityService.updateAspect(entityUrn, "corpuser", aspectName, corpUserInfoSpec, writeAspect, TEST_AUDIT_STAMP, 1, + _entityServiceImpl.updateAspect(entityUrn, "corpuser", aspectName, corpUserInfoSpec, writeAspect, TEST_AUDIT_STAMP, 1, true); VersionedAspect writtenVersionedAspect = new VersionedAspect(); writtenVersionedAspect.setAspect(Aspect.create(writeAspect)); writtenVersionedAspect.setVersion(1); - VersionedAspect readAspect1 = _entityService.getVersionedAspect(entityUrn, aspectName, 1); + VersionedAspect readAspect1 = _entityServiceImpl.getVersionedAspect(entityUrn, aspectName, 1); assertTrue(DataTemplateUtil.areEqual(writtenVersionedAspect, readAspect1)); verify(_mockProducer, times(1)).produceMetadataChangeLog(Mockito.eq(entityUrn), Mockito.eq(corpUserInfoSpec), Mockito.any()); - VersionedAspect readAspect2 = _entityService.getVersionedAspect(entityUrn, aspectName, -1); + VersionedAspect readAspect2 = _entityServiceImpl.getVersionedAspect(entityUrn, aspectName, -1); assertTrue(DataTemplateUtil.areEqual(writtenVersionedAspect, readAspect2)); - VersionedAspect readAspectVersion0 = _entityService.getVersionedAspect(entityUrn, aspectName, 0); + VersionedAspect readAspectVersion0 = _entityServiceImpl.getVersionedAspect(entityUrn, aspectName, 0); assertFalse(DataTemplateUtil.areEqual(writtenVersionedAspect, readAspectVersion0)); verifyNoMoreInteractions(_mockProducer); @@ -768,19 +768,19 @@ public void testRollbackAspect() throws AssertionError { // Ingest CorpUserInfo Aspect #1 CorpUserInfo writeAspect1 = AspectGenerationUtils.createCorpUserInfo("email@test.com"); - _entityService.ingestAspect(entityUrn1, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspect(entityUrn1, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #2 CorpUserInfo writeAspect2 = AspectGenerationUtils.createCorpUserInfo("email2@test.com"); - _entityService.ingestAspect(entityUrn2, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspect(entityUrn2, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #3 CorpUserInfo writeAspect3 = AspectGenerationUtils.createCorpUserInfo("email3@test.com"); - _entityService.ingestAspect(entityUrn3, aspectName, writeAspect3, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspect(entityUrn3, aspectName, writeAspect3, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #1 Overwrite CorpUserInfo writeAspect1Overwrite = AspectGenerationUtils.createCorpUserInfo("email1.overwrite@test.com"); - _entityService.ingestAspect(entityUrn1, aspectName, writeAspect1Overwrite, TEST_AUDIT_STAMP, metadata2); + _entityServiceImpl.ingestAspect(entityUrn1, aspectName, writeAspect1Overwrite, TEST_AUDIT_STAMP, metadata2); // this should no-op since this run has been overwritten AspectRowSummary rollbackOverwrittenAspect = new AspectRowSummary(); @@ -788,13 +788,13 @@ public void testRollbackAspect() throws AssertionError { rollbackOverwrittenAspect.setAspectName(aspectName); rollbackOverwrittenAspect.setUrn(entityUrn1.toString()); - _entityService.rollbackRun(ImmutableList.of(rollbackOverwrittenAspect), "run-123", true); + _entityServiceImpl.rollbackRun(ImmutableList.of(rollbackOverwrittenAspect), "run-123", true); // assert nothing was deleted - RecordTemplate readAspectOriginal = _entityService.getAspect(entityUrn1, aspectName, 1); + RecordTemplate readAspectOriginal = _entityServiceImpl.getAspect(entityUrn1, aspectName, 1); assertTrue(DataTemplateUtil.areEqual(writeAspect1, readAspectOriginal)); - RecordTemplate readAspectOverwrite = _entityService.getAspect(entityUrn1, aspectName, 0); + RecordTemplate readAspectOverwrite = _entityServiceImpl.getAspect(entityUrn1, aspectName, 0); assertTrue(DataTemplateUtil.areEqual(writeAspect1Overwrite, readAspectOverwrite)); // this should delete the most recent aspect @@ -803,10 +803,10 @@ public void testRollbackAspect() throws AssertionError { rollbackRecentAspect.setAspectName(aspectName); rollbackRecentAspect.setUrn(entityUrn1.toString()); - _entityService.rollbackRun(ImmutableList.of(rollbackOverwrittenAspect), "run-456", true); + _entityServiceImpl.rollbackRun(ImmutableList.of(rollbackOverwrittenAspect), "run-456", true); // assert the new most recent aspect is the original one - RecordTemplate readNewRecentAspect = _entityService.getAspect(entityUrn1, aspectName, 0); + RecordTemplate readNewRecentAspect = _entityServiceImpl.getAspect(entityUrn1, aspectName, 0); assertTrue(DataTemplateUtil.areEqual(writeAspect1, readNewRecentAspect)); } @@ -818,18 +818,18 @@ public void testRollbackKey() throws AssertionError { SystemMetadata metadata2 = AspectGenerationUtils.createSystemMetadata(1635792689, "run-456"); String aspectName = AspectGenerationUtils.getAspectName(new CorpUserInfo()); - String keyAspectName = _entityService.getKeyAspectName(entityUrn1); + String keyAspectName = _entityServiceImpl.getKeyAspectName(entityUrn1); // Ingest CorpUserInfo Aspect #1 CorpUserInfo writeAspect1 = AspectGenerationUtils.createCorpUserInfo("email@test.com"); - _entityService.ingestAspect(entityUrn1, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspect(entityUrn1, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); - RecordTemplate writeKey1 = _entityService.buildKeyAspect(entityUrn1); - _entityService.ingestAspect(entityUrn1, keyAspectName, writeKey1, TEST_AUDIT_STAMP, metadata1); + RecordTemplate writeKey1 = _entityServiceImpl.buildKeyAspect(entityUrn1); + _entityServiceImpl.ingestAspect(entityUrn1, keyAspectName, writeKey1, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #1 Overwrite CorpUserInfo writeAspect1Overwrite = AspectGenerationUtils.createCorpUserInfo("email1.overwrite@test.com"); - _entityService.ingestAspect(entityUrn1, aspectName, writeAspect1Overwrite, TEST_AUDIT_STAMP, metadata2); + _entityServiceImpl.ingestAspect(entityUrn1, aspectName, writeAspect1Overwrite, TEST_AUDIT_STAMP, metadata2); // this should no-op since the key should have been written in the furst run AspectRowSummary rollbackKeyWithWrongRunId = new AspectRowSummary(); @@ -837,13 +837,13 @@ public void testRollbackKey() throws AssertionError { rollbackKeyWithWrongRunId.setAspectName("corpUserKey"); rollbackKeyWithWrongRunId.setUrn(entityUrn1.toString()); - _entityService.rollbackRun(ImmutableList.of(rollbackKeyWithWrongRunId), "run-456", true); + _entityServiceImpl.rollbackRun(ImmutableList.of(rollbackKeyWithWrongRunId), "run-456", true); // assert nothing was deleted - RecordTemplate readAspectOriginal = _entityService.getAspect(entityUrn1, aspectName, 1); + RecordTemplate readAspectOriginal = _entityServiceImpl.getAspect(entityUrn1, aspectName, 1); assertTrue(DataTemplateUtil.areEqual(writeAspect1, readAspectOriginal)); - RecordTemplate readAspectOverwrite = _entityService.getAspect(entityUrn1, aspectName, 0); + RecordTemplate readAspectOverwrite = _entityServiceImpl.getAspect(entityUrn1, aspectName, 0); assertTrue(DataTemplateUtil.areEqual(writeAspect1Overwrite, readAspectOverwrite)); // this should delete the most recent aspect @@ -852,10 +852,10 @@ public void testRollbackKey() throws AssertionError { rollbackKeyWithCorrectRunId.setAspectName("corpUserKey"); rollbackKeyWithCorrectRunId.setUrn(entityUrn1.toString()); - _entityService.rollbackRun(ImmutableList.of(rollbackKeyWithCorrectRunId), "run-123", true); + _entityServiceImpl.rollbackRun(ImmutableList.of(rollbackKeyWithCorrectRunId), "run-123", true); // assert the new most recent aspect is null - RecordTemplate readNewRecentAspect = _entityService.getAspect(entityUrn1, aspectName, 0); + RecordTemplate readNewRecentAspect = _entityServiceImpl.getAspect(entityUrn1, aspectName, 0); assertTrue(DataTemplateUtil.areEqual(null, readNewRecentAspect)); } @@ -869,26 +869,26 @@ public void testRollbackUrn() throws AssertionError { SystemMetadata metadata2 = AspectGenerationUtils.createSystemMetadata(1635792689, "run-456"); String aspectName = AspectGenerationUtils.getAspectName(new CorpUserInfo()); - String keyAspectName = _entityService.getKeyAspectName(entityUrn1); + String keyAspectName = _entityServiceImpl.getKeyAspectName(entityUrn1); // Ingest CorpUserInfo Aspect #1 CorpUserInfo writeAspect1 = AspectGenerationUtils.createCorpUserInfo("email@test.com"); - _entityService.ingestAspect(entityUrn1, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspect(entityUrn1, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); - RecordTemplate writeKey1 = _entityService.buildKeyAspect(entityUrn1); - _entityService.ingestAspect(entityUrn1, keyAspectName, writeKey1, TEST_AUDIT_STAMP, metadata1); + RecordTemplate writeKey1 = _entityServiceImpl.buildKeyAspect(entityUrn1); + _entityServiceImpl.ingestAspect(entityUrn1, keyAspectName, writeKey1, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #2 CorpUserInfo writeAspect2 = AspectGenerationUtils.createCorpUserInfo("email2@test.com"); - _entityService.ingestAspect(entityUrn2, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspect(entityUrn2, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #3 CorpUserInfo writeAspect3 = AspectGenerationUtils.createCorpUserInfo("email3@test.com"); - _entityService.ingestAspect(entityUrn3, aspectName, writeAspect3, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspect(entityUrn3, aspectName, writeAspect3, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #1 Overwrite CorpUserInfo writeAspect1Overwrite = AspectGenerationUtils.createCorpUserInfo("email1.overwrite@test.com"); - _entityService.ingestAspect(entityUrn1, aspectName, writeAspect1Overwrite, TEST_AUDIT_STAMP, metadata2); + _entityServiceImpl.ingestAspect(entityUrn1, aspectName, writeAspect1Overwrite, TEST_AUDIT_STAMP, metadata2); // this should no-op since the key should have been written in the furst run AspectRowSummary rollbackKeyWithWrongRunId = new AspectRowSummary(); @@ -897,13 +897,13 @@ public void testRollbackUrn() throws AssertionError { rollbackKeyWithWrongRunId.setUrn(entityUrn1.toString()); // this should delete all related aspects - _entityService.deleteUrn(UrnUtils.getUrn("urn:li:corpuser:test1")); + _entityServiceImpl.deleteUrn(UrnUtils.getUrn("urn:li:corpuser:test1")); // assert the new most recent aspect is null - RecordTemplate readNewRecentAspect = _entityService.getAspect(entityUrn1, aspectName, 0); + RecordTemplate readNewRecentAspect = _entityServiceImpl.getAspect(entityUrn1, aspectName, 0); assertTrue(DataTemplateUtil.areEqual(null, readNewRecentAspect)); - RecordTemplate deletedKeyAspect = _entityService.getAspect(entityUrn1, "corpUserKey", 0); + RecordTemplate deletedKeyAspect = _entityServiceImpl.getAspect(entityUrn1, "corpUserKey", 0); assertTrue(DataTemplateUtil.areEqual(null, deletedKeyAspect)); } @@ -919,8 +919,8 @@ public void testIngestGetLatestAspect() throws AssertionError { SystemMetadata metadata2 = AspectGenerationUtils.createSystemMetadata(1635792689, "run-456"); // Validate retrieval of CorpUserInfo Aspect #1 - _entityService.ingestAspect(entityUrn, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); - RecordTemplate readAspect1 = _entityService.getLatestAspect(entityUrn, aspectName); + _entityServiceImpl.ingestAspect(entityUrn, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); + RecordTemplate readAspect1 = _entityServiceImpl.getLatestAspect(entityUrn, aspectName); assertTrue(DataTemplateUtil.areEqual(writeAspect1, readAspect1)); ArgumentCaptor mclCaptor = ArgumentCaptor.forClass(MetadataChangeLog.class); @@ -942,8 +942,8 @@ public void testIngestGetLatestAspect() throws AssertionError { CorpUserInfo writeAspect2 = AspectGenerationUtils.createCorpUserInfo("email2@test.com"); // Validate retrieval of CorpUserInfo Aspect #2 - _entityService.ingestAspect(entityUrn, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata2); - RecordTemplate readAspect2 = _entityService.getLatestAspect(entityUrn, aspectName); + _entityServiceImpl.ingestAspect(entityUrn, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata2); + RecordTemplate readAspect2 = _entityServiceImpl.getLatestAspect(entityUrn, aspectName); EntityAspect readAspectDao1 = _aspectDao.getAspect(entityUrn.toString(), aspectName, 1); EntityAspect readAspectDao2 = _aspectDao.getAspect(entityUrn.toString(), aspectName, 0); @@ -976,16 +976,16 @@ public void testIngestGetLatestEnvelopedAspect() throws Exception { SystemMetadata metadata2 = AspectGenerationUtils.createSystemMetadata(1635792689, "run-456"); // Validate retrieval of CorpUserInfo Aspect #1 - _entityService.ingestAspect(entityUrn, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); - EnvelopedAspect readAspect1 = _entityService.getLatestEnvelopedAspect("corpuser", entityUrn, aspectName); + _entityServiceImpl.ingestAspect(entityUrn, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); + EnvelopedAspect readAspect1 = _entityServiceImpl.getLatestEnvelopedAspect("corpuser", entityUrn, aspectName); assertTrue(DataTemplateUtil.areEqual(writeAspect1, new CorpUserInfo(readAspect1.getValue().data()))); // Ingest CorpUserInfo Aspect #2 CorpUserInfo writeAspect2 = AspectGenerationUtils.createCorpUserInfo("email2@test.com"); // Validate retrieval of CorpUserInfo Aspect #2 - _entityService.ingestAspect(entityUrn, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata2); - EnvelopedAspect readAspect2 = _entityService.getLatestEnvelopedAspect("corpuser", entityUrn, aspectName); + _entityServiceImpl.ingestAspect(entityUrn, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata2); + EnvelopedAspect readAspect2 = _entityServiceImpl.getLatestEnvelopedAspect("corpuser", entityUrn, aspectName); EntityAspect readAspectDao1 = _aspectDao.getAspect(entityUrn.toString(), aspectName, 1); EntityAspect readAspectDao2 = _aspectDao.getAspect(entityUrn.toString(), aspectName, 0); @@ -1018,8 +1018,8 @@ public void testIngestSameAspect() throws AssertionError { SystemMetadata metadata3 = AspectGenerationUtils.createSystemMetadata(1635792689, "run-123"); // Validate retrieval of CorpUserInfo Aspect #1 - _entityService.ingestAspect(entityUrn, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); - RecordTemplate readAspect1 = _entityService.getLatestAspect(entityUrn, aspectName); + _entityServiceImpl.ingestAspect(entityUrn, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); + RecordTemplate readAspect1 = _entityServiceImpl.getLatestAspect(entityUrn, aspectName); assertTrue(DataTemplateUtil.areEqual(writeAspect1, readAspect1)); ArgumentCaptor mclCaptor = ArgumentCaptor.forClass(MetadataChangeLog.class); @@ -1041,8 +1041,8 @@ public void testIngestSameAspect() throws AssertionError { CorpUserInfo writeAspect2 = AspectGenerationUtils.createCorpUserInfo("email@test.com"); // Validate retrieval of CorpUserInfo Aspect #2 - _entityService.ingestAspect(entityUrn, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata2); - RecordTemplate readAspect2 = _entityService.getLatestAspect(entityUrn, aspectName); + _entityServiceImpl.ingestAspect(entityUrn, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata2); + RecordTemplate readAspect2 = _entityServiceImpl.getLatestAspect(entityUrn, aspectName); EntityAspect readAspectDao2 = _aspectDao.getAspect(entityUrn.toString(), aspectName, ASPECT_LATEST_VERSION); assertTrue(DataTemplateUtil.areEqual(writeAspect2, readAspect2)); @@ -1069,23 +1069,23 @@ public void testRetention() throws AssertionError { // Ingest CorpUserInfo Aspect CorpUserInfo writeAspect1 = AspectGenerationUtils.createCorpUserInfo("email@test.com"); - _entityService.ingestAspect(entityUrn, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspect(entityUrn, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); CorpUserInfo writeAspect1a = AspectGenerationUtils.createCorpUserInfo("email_a@test.com"); - _entityService.ingestAspect(entityUrn, aspectName, writeAspect1a, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspect(entityUrn, aspectName, writeAspect1a, TEST_AUDIT_STAMP, metadata1); CorpUserInfo writeAspect1b = AspectGenerationUtils.createCorpUserInfo("email_b@test.com"); - _entityService.ingestAspect(entityUrn, aspectName, writeAspect1b, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspect(entityUrn, aspectName, writeAspect1b, TEST_AUDIT_STAMP, metadata1); String aspectName2 = AspectGenerationUtils.getAspectName(new Status()); // Ingest Status Aspect Status writeAspect2 = new Status().setRemoved(true); - _entityService.ingestAspect(entityUrn, aspectName2, writeAspect2, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspect(entityUrn, aspectName2, writeAspect2, TEST_AUDIT_STAMP, metadata1); Status writeAspect2a = new Status().setRemoved(false); - _entityService.ingestAspect(entityUrn, aspectName2, writeAspect2a, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspect(entityUrn, aspectName2, writeAspect2a, TEST_AUDIT_STAMP, metadata1); Status writeAspect2b = new Status().setRemoved(true); - _entityService.ingestAspect(entityUrn, aspectName2, writeAspect2b, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspect(entityUrn, aspectName2, writeAspect2b, TEST_AUDIT_STAMP, metadata1); - assertEquals(_entityService.getAspect(entityUrn, aspectName, 1), writeAspect1); - assertEquals(_entityService.getAspect(entityUrn, aspectName2, 1), writeAspect2); + assertEquals(_entityServiceImpl.getAspect(entityUrn, aspectName, 1), writeAspect1); + assertEquals(_entityServiceImpl.getAspect(entityUrn, aspectName2, 1), writeAspect2); _retentionService.setRetention(null, null, new DataHubRetentionConfig().setRetention( new Retention().setVersion(new VersionBasedRetention().setMaxVersions(2)))); @@ -1094,13 +1094,13 @@ public void testRetention() throws AssertionError { // Ingest CorpUserInfo Aspect again CorpUserInfo writeAspect1c = AspectGenerationUtils.createCorpUserInfo("email_c@test.com"); - _entityService.ingestAspect(entityUrn, aspectName, writeAspect1c, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspect(entityUrn, aspectName, writeAspect1c, TEST_AUDIT_STAMP, metadata1); // Ingest Status Aspect again Status writeAspect2c = new Status().setRemoved(false); - _entityService.ingestAspect(entityUrn, aspectName2, writeAspect2c, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspect(entityUrn, aspectName2, writeAspect2c, TEST_AUDIT_STAMP, metadata1); - assertNull(_entityService.getAspect(entityUrn, aspectName, 1)); - assertEquals(_entityService.getAspect(entityUrn, aspectName2, 1), writeAspect2); + assertNull(_entityServiceImpl.getAspect(entityUrn, aspectName, 1)); + assertEquals(_entityServiceImpl.getAspect(entityUrn, aspectName2, 1), writeAspect2); // Reset retention policies _retentionService.setRetention(null, null, new DataHubRetentionConfig().setRetention( @@ -1108,8 +1108,8 @@ public void testRetention() throws AssertionError { _retentionService.deleteRetention("corpuser", "status"); // Invoke batch apply _retentionService.batchApplyRetention(null, null); - assertEquals(_entityService.listLatestAspects(entityUrn.getEntityType(), aspectName, 0, 10).getTotalCount(), 1); - assertEquals(_entityService.listLatestAspects(entityUrn.getEntityType(), aspectName2, 0, 10).getTotalCount(), 1); + assertEquals(_entityServiceImpl.listLatestAspects(entityUrn.getEntityType(), aspectName, 0, 10).getTotalCount(), 1); + assertEquals(_entityServiceImpl.listLatestAspects(entityUrn.getEntityType(), aspectName2, 0, 10).getTotalCount(), 1); } @Test @@ -1122,29 +1122,29 @@ public void testIngestAspectIfNotPresent() throws AssertionError { // Ingest CorpUserInfo Aspect CorpUserInfo writeAspect1 = AspectGenerationUtils.createCorpUserInfo("email@test.com"); - _entityService.ingestAspectIfNotPresent(entityUrn, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspectIfNotPresent(entityUrn, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); CorpUserInfo writeAspect1a = AspectGenerationUtils.createCorpUserInfo("email_a@test.com"); - _entityService.ingestAspectIfNotPresent(entityUrn, aspectName, writeAspect1a, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspectIfNotPresent(entityUrn, aspectName, writeAspect1a, TEST_AUDIT_STAMP, metadata1); CorpUserInfo writeAspect1b = AspectGenerationUtils.createCorpUserInfo("email_b@test.com"); - _entityService.ingestAspectIfNotPresent(entityUrn, aspectName, writeAspect1b, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspectIfNotPresent(entityUrn, aspectName, writeAspect1b, TEST_AUDIT_STAMP, metadata1); String aspectName2 = AspectGenerationUtils.getAspectName(new Status()); // Ingest Status Aspect Status writeAspect2 = new Status().setRemoved(true); - _entityService.ingestAspectIfNotPresent(entityUrn, aspectName2, writeAspect2, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspectIfNotPresent(entityUrn, aspectName2, writeAspect2, TEST_AUDIT_STAMP, metadata1); Status writeAspect2a = new Status().setRemoved(false); - _entityService.ingestAspectIfNotPresent(entityUrn, aspectName2, writeAspect2a, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspectIfNotPresent(entityUrn, aspectName2, writeAspect2a, TEST_AUDIT_STAMP, metadata1); Status writeAspect2b = new Status().setRemoved(true); - _entityService.ingestAspectIfNotPresent(entityUrn, aspectName2, writeAspect2b, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspectIfNotPresent(entityUrn, aspectName2, writeAspect2b, TEST_AUDIT_STAMP, metadata1); - assertEquals(_entityService.getAspect(entityUrn, aspectName, 0), writeAspect1); - assertEquals(_entityService.getAspect(entityUrn, aspectName2, 0), writeAspect2); + assertEquals(_entityServiceImpl.getAspect(entityUrn, aspectName, 0), writeAspect1); + assertEquals(_entityServiceImpl.getAspect(entityUrn, aspectName2, 0), writeAspect2); - assertNull(_entityService.getAspect(entityUrn, aspectName, 1)); - assertNull(_entityService.getAspect(entityUrn, aspectName2, 1)); + assertNull(_entityServiceImpl.getAspect(entityUrn, aspectName, 1)); + assertNull(_entityServiceImpl.getAspect(entityUrn, aspectName2, 1)); - assertEquals(_entityService.listLatestAspects(entityUrn.getEntityType(), aspectName, 0, 10).getTotalCount(), 1); - assertEquals(_entityService.listLatestAspects(entityUrn.getEntityType(), aspectName2, 0, 10).getTotalCount(), 1); + assertEquals(_entityServiceImpl.listLatestAspects(entityUrn.getEntityType(), aspectName, 0, 10).getTotalCount(), 1); + assertEquals(_entityServiceImpl.listLatestAspects(entityUrn.getEntityType(), aspectName2, 0, 10).getTotalCount(), 1); } /** @@ -1173,7 +1173,7 @@ public void testRestoreIndices() throws Exception { SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(); - _entityService.ingestAspects(entityUrn, pairToIngest, TEST_AUDIT_STAMP, metadata1); + _entityServiceImpl.ingestAspects(entityUrn, pairToIngest, TEST_AUDIT_STAMP, metadata1); clearInvocations(_mockProducer); @@ -1184,7 +1184,7 @@ public void testRestoreIndices() throws Exception { args.setBatchDelayMs(1L); args.setNumThreads(1); args.setUrn(urnStr); - _entityService.restoreIndices(args, obj -> { + _entityServiceImpl.restoreIndices(args, obj -> { }); ArgumentCaptor mclCaptor = ArgumentCaptor.forClass(MetadataChangeLog.class); @@ -1203,12 +1203,12 @@ public void testRestoreIndices() throws Exception { public void testValidateUrn() throws Exception { // Valid URN Urn validTestUrn = new Urn("li", "corpuser", new TupleKey("testKey")); - _entityService.validateUrn(validTestUrn); + _entityServiceImpl.validateUrn(validTestUrn); // URN with trailing whitespace Urn testUrnWithTrailingWhitespace = new Urn("li", "corpuser", new TupleKey("testKey ")); try { - _entityService.validateUrn(testUrnWithTrailingWhitespace); + _entityServiceImpl.validateUrn(testUrnWithTrailingWhitespace); Assert.fail("Should have raised IllegalArgumentException for URN with trailing whitespace"); } catch (IllegalArgumentException e) { assertEquals(e.getMessage(), "Error: cannot provide an URN with leading or trailing whitespace"); @@ -1219,7 +1219,7 @@ public void testValidateUrn() throws Exception { Urn testUrnTooLong = new Urn("li", "corpuser", new TupleKey(stringTooLong)); try { - _entityService.validateUrn(testUrnTooLong); + _entityServiceImpl.validateUrn(testUrnTooLong); Assert.fail("Should have raised IllegalArgumentException for URN too long"); } catch (IllegalArgumentException e) { assertEquals(e.getMessage(), "Error: cannot provide an URN longer than 512 bytes (when URL encoded)"); @@ -1235,9 +1235,9 @@ public void testValidateUrn() throws Exception { Urn testUrnTooLongWhenEncoded = new Urn("li", "corpUser", new TupleKey(buildStringTooLongWhenEncoded.toString())); Urn testUrnSameLengthWhenEncoded = new Urn("li", "corpUser", new TupleKey(buildStringSameLengthWhenEncoded.toString())); // Same length when encoded should be allowed, the encoded one should not be - _entityService.validateUrn(testUrnSameLengthWhenEncoded); + _entityServiceImpl.validateUrn(testUrnSameLengthWhenEncoded); try { - _entityService.validateUrn(testUrnTooLongWhenEncoded); + _entityServiceImpl.validateUrn(testUrnTooLongWhenEncoded); Assert.fail("Should have raised IllegalArgumentException for URN too long"); } catch (IllegalArgumentException e) { assertEquals(e.getMessage(), "Error: cannot provide an URN longer than 512 bytes (when URL encoded)"); @@ -1246,9 +1246,9 @@ public void testValidateUrn() throws Exception { // Urn containing disallowed character Urn testUrnSpecialCharValid = new Urn("li", "corpUser", new TupleKey("bob␇")); Urn testUrnSpecialCharInvalid = new Urn("li", "corpUser", new TupleKey("bob␟")); - _entityService.validateUrn(testUrnSpecialCharValid); + _entityServiceImpl.validateUrn(testUrnSpecialCharValid); try { - _entityService.validateUrn(testUrnSpecialCharInvalid); + _entityServiceImpl.validateUrn(testUrnSpecialCharInvalid); Assert.fail("Should have raised IllegalArgumentException for URN containing the illegal char"); } catch (IllegalArgumentException e) { assertEquals(e.getMessage(), "Error: URN cannot contain ␟ character"); @@ -1256,7 +1256,7 @@ public void testValidateUrn() throws Exception { Urn urnWithMismatchedParens = new Urn("li", "corpuser", new TupleKey("test(Key")); try { - _entityService.validateUrn(urnWithMismatchedParens); + _entityServiceImpl.validateUrn(urnWithMismatchedParens); Assert.fail("Should have raised IllegalArgumentException for URN with mismatched parens"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("mismatched paren nesting")); @@ -1264,18 +1264,18 @@ public void testValidateUrn() throws Exception { Urn invalidType = new Urn("li", "fakeMadeUpType", new TupleKey("testKey")); try { - _entityService.validateUrn(invalidType); + _entityServiceImpl.validateUrn(invalidType); Assert.fail("Should have raised IllegalArgumentException for URN with non-existent entity type"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("Failed to find entity with name fakeMadeUpType")); } Urn validFabricType = new Urn("li", "dataset", new TupleKey("urn:li:dataPlatform:foo", "bar", "PROD")); - _entityService.validateUrn(validFabricType); + _entityServiceImpl.validateUrn(validFabricType); Urn invalidFabricType = new Urn("li", "dataset", new TupleKey("urn:li:dataPlatform:foo", "bar", "prod")); try { - _entityService.validateUrn(invalidFabricType); + _entityServiceImpl.validateUrn(invalidFabricType); Assert.fail("Should have raised IllegalArgumentException for URN with invalid fabric type"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains(invalidFabricType.toString())); @@ -1283,7 +1283,7 @@ public void testValidateUrn() throws Exception { Urn urnEndingInComma = new Urn("li", "dataset", new TupleKey("urn:li:dataPlatform:foo", "bar", "PROD", "")); try { - _entityService.validateUrn(urnEndingInComma); + _entityServiceImpl.validateUrn(urnEndingInComma); Assert.fail("Should have raised IllegalArgumentException for URN ending in comma"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains(urnEndingInComma.toString())); @@ -1312,7 +1312,7 @@ public void testUIPreProcessedProposal() throws Exception { genericAspect.setValue(ByteString.unsafeWrap(datasetPropertiesSerialized)); genericAspect.setContentType("application/json"); gmce.setAspect(genericAspect); - _entityService.ingestProposal(gmce, TEST_AUDIT_STAMP, false); + _entityServiceImpl.ingestProposal(gmce, TEST_AUDIT_STAMP, false); ArgumentCaptor captor = ArgumentCaptor.forClass(MetadataChangeLog.class); verify(_mockProducer, times(1)).produceMetadataChangeLog(Mockito.eq(entityUrn), Mockito.any(), captor.capture()); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphServiceTest.java index 91626cef9d409..1717e466359d3 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphServiceTest.java @@ -1,12 +1,12 @@ package com.linkedin.metadata.graph.elastic; +import com.linkedin.metadata.config.search.GraphQueryConfiguration; import com.linkedin.common.FabricType; import com.linkedin.common.urn.DataPlatformUrn; import com.linkedin.common.urn.DatasetUrn; import com.linkedin.common.urn.TagUrn; import com.linkedin.common.urn.Urn; import com.linkedin.metadata.ESTestConfiguration; -import com.linkedin.metadata.config.search.GraphQueryConfiguration; import com.linkedin.metadata.graph.Edge; import com.linkedin.metadata.graph.EntityLineageResult; import com.linkedin.metadata.graph.GraphService; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchServiceTest.java index 8498004fc4a93..e8872691772de 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchServiceTest.java @@ -1,5 +1,8 @@ package com.linkedin.metadata.search; +import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; +import com.linkedin.metadata.config.cache.SearchLineageCacheConfiguration; +import com.linkedin.metadata.config.search.SearchConfiguration; import com.datahub.test.Snapshot; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -15,9 +18,6 @@ import com.linkedin.data.template.LongMap; import com.linkedin.metadata.ESTestConfiguration; import com.linkedin.metadata.TestEntityUtil; -import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; -import com.linkedin.metadata.config.cache.SearchLineageCacheConfiguration; -import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.graph.EntityLineageResult; import com.linkedin.metadata.graph.GraphService; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTest.java index 8bae3c9e91af5..2405b3b3f6e27 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search; +import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; +import com.linkedin.metadata.config.search.SearchConfiguration; import com.datahub.test.Snapshot; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -8,8 +10,6 @@ import com.linkedin.common.urn.Urn; import com.linkedin.data.template.StringArray; import com.linkedin.metadata.ESTestConfiguration; -import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; -import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.models.registry.SnapshotEntityRegistry; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java index c21c5fc92e960..0e8d881b70791 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.search.elasticsearch; +import com.linkedin.metadata.config.search.SearchConfiguration; import com.datahub.test.Snapshot; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -8,7 +9,6 @@ import com.linkedin.metadata.ESTestConfiguration; import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor; import com.linkedin.metadata.browse.BrowseResult; -import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.models.registry.SnapshotEntityRegistry; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilderTest.java index f1f910e93376b..2416280cb8f93 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilderTest.java @@ -1,8 +1,8 @@ package com.linkedin.metadata.search.elasticsearch.indexbuilder; +import com.linkedin.metadata.config.search.ElasticSearchConfiguration; import com.google.common.collect.ImmutableMap; import com.linkedin.metadata.ESTestConfiguration; -import com.linkedin.metadata.config.search.ElasticSearchConfiguration; import com.linkedin.metadata.systemmetadata.SystemMetadataMappingsBuilder; import com.linkedin.metadata.version.GitVersion; import java.util.Optional; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAOTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAOTest.java index eb3da419651e6..b506051e9bb5d 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAOTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAOTest.java @@ -1,11 +1,11 @@ package com.linkedin.metadata.search.elasticsearch.query; +import com.linkedin.metadata.config.search.SearchConfiguration; import com.datahub.test.Snapshot; import com.google.common.collect.ImmutableList; import com.linkedin.data.template.LongMap; import com.linkedin.data.template.StringArray; import com.linkedin.metadata.ESSampleDataFixture; -import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.models.registry.SnapshotEntityRegistry; import com.linkedin.metadata.query.filter.Condition; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilderTest.java index 914911677c2c9..76160eb29af4e 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilderTest.java @@ -1,7 +1,7 @@ package com.linkedin.metadata.search.elasticsearch.query.request; -import com.google.common.collect.ImmutableList; import com.linkedin.metadata.config.search.SearchConfiguration; +import com.google.common.collect.ImmutableList; import com.linkedin.metadata.models.annotation.SearchableAnnotation; import java.util.Collections; import java.util.List; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/CustomizedQueryHandlerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/CustomizedQueryHandlerTest.java index 3f63e6b43a871..3dad9c59c6b53 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/CustomizedQueryHandlerTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/CustomizedQueryHandlerTest.java @@ -1,11 +1,12 @@ package com.linkedin.metadata.search.elasticsearch.query.request; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; import com.linkedin.metadata.config.search.CustomConfiguration; +import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.config.search.custom.BoolQueryConfiguration; -import com.linkedin.metadata.config.search.custom.QueryConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; +import com.linkedin.metadata.config.search.custom.QueryConfiguration; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; import org.elasticsearch.common.lucene.search.function.CombineFunction; import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery; import org.elasticsearch.index.query.MatchAllQueryBuilder; @@ -35,6 +36,10 @@ public class CustomizedQueryHandlerTest { throw new RuntimeException(e); } } + public static final SearchQueryBuilder SEARCH_QUERY_BUILDER; + static { + SEARCH_QUERY_BUILDER = new SearchQueryBuilder(new SearchConfiguration(), TEST_CONFIG); + } private static final List EXPECTED_CONFIGURATION = List.of( QueryConfiguration.builder() .queryRegex("[*]|") @@ -132,7 +137,8 @@ public void functionScoreQueryBuilderTest() { /* * Test select star */ - FunctionScoreQueryBuilder selectStarTest = test.lookupQueryConfig("*").get().functionScoreQueryBuilder(inputQuery); + FunctionScoreQueryBuilder selectStarTest = SEARCH_QUERY_BUILDER.functionScoreQueryBuilder(test.lookupQueryConfig("*").get(), + inputQuery); FunctionScoreQueryBuilder.FilterFunctionBuilder[] expectedSelectStarScoreFunctions = { new FunctionScoreQueryBuilder.FilterFunctionBuilder( @@ -156,7 +162,7 @@ public void functionScoreQueryBuilderTest() { /* * Test default (non-select start) */ - FunctionScoreQueryBuilder defaultTest = test.lookupQueryConfig("foobar").get().functionScoreQueryBuilder(inputQuery); + FunctionScoreQueryBuilder defaultTest = SEARCH_QUERY_BUILDER.functionScoreQueryBuilder(test.lookupQueryConfig("foobar").get(), inputQuery); FunctionScoreQueryBuilder.FilterFunctionBuilder[] expectedDefaultScoreFunctions = { new FunctionScoreQueryBuilder.FilterFunctionBuilder( diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilderTest.java index e35fc4528ca60..a2ec396c34b2d 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilderTest.java @@ -1,5 +1,10 @@ package com.linkedin.metadata.search.elasticsearch.query.request; +import com.linkedin.metadata.config.search.CustomConfiguration; +import com.linkedin.metadata.config.search.ExactMatchConfiguration; +import com.linkedin.metadata.config.search.PartialConfiguration; +import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; import com.google.common.collect.ImmutableList; import com.linkedin.metadata.TestEntitySpecBuilder; @@ -9,11 +14,6 @@ import java.util.Map; import java.util.stream.Collectors; -import com.linkedin.metadata.config.search.CustomConfiguration; -import com.linkedin.metadata.config.search.ExactMatchConfiguration; -import com.linkedin.metadata.config.search.PartialConfiguration; -import com.linkedin.metadata.config.search.SearchConfiguration; -import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.util.Pair; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandlerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandlerTest.java index 095c99370e8d1..f85739d20fc65 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandlerTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandlerTest.java @@ -1,5 +1,8 @@ package com.linkedin.metadata.search.elasticsearch.query.request; +import com.linkedin.metadata.config.search.ExactMatchConfiguration; +import com.linkedin.metadata.config.search.PartialConfiguration; +import com.linkedin.metadata.config.search.SearchConfiguration; import com.google.common.collect.ImmutableList; import com.linkedin.data.template.StringArray; import com.linkedin.metadata.ESTestConfiguration; @@ -15,9 +18,6 @@ import java.util.Set; import java.util.stream.Collectors; -import com.linkedin.metadata.config.search.ExactMatchConfiguration; -import com.linkedin.metadata.config.search.PartialConfiguration; -import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.query.SearchFlags; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/timeline/CassandraTimelineServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/timeline/CassandraTimelineServiceTest.java index fd8cdec73a191..407d2ae684ede 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/timeline/CassandraTimelineServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/timeline/CassandraTimelineServiceTest.java @@ -1,9 +1,9 @@ package com.linkedin.metadata.timeline; +import com.linkedin.metadata.config.PreProcessHooks; import com.datastax.oss.driver.api.core.CqlSession; import com.linkedin.metadata.CassandraTestUtils; -import com.linkedin.metadata.config.PreProcessHooks; -import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.EntityServiceImpl; import com.linkedin.metadata.entity.cassandra.CassandraAspectDao; import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.models.registry.EntityRegistryException; @@ -54,7 +54,7 @@ private void configureComponents() { _mockProducer = mock(EventProducer.class); PreProcessHooks preProcessHooks = new PreProcessHooks(); preProcessHooks.setUiEnabled(true); - _entityService = new EntityService(_aspectDao, _mockProducer, _testEntityRegistry, true, + _entityServiceImpl = new EntityServiceImpl(_aspectDao, _mockProducer, _testEntityRegistry, true, _mockUpdateIndicesService, preProcessHooks); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/timeline/EbeanTimelineServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/timeline/EbeanTimelineServiceTest.java index 0ea2f6bb124d9..b431f786cd50a 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/timeline/EbeanTimelineServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/timeline/EbeanTimelineServiceTest.java @@ -1,8 +1,8 @@ package com.linkedin.metadata.timeline; -import com.linkedin.metadata.EbeanTestUtils; import com.linkedin.metadata.config.PreProcessHooks; -import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.EbeanTestUtils; +import com.linkedin.metadata.entity.EntityServiceImpl; import com.linkedin.metadata.entity.ebean.EbeanAspectDao; import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.models.registry.EntityRegistryException; @@ -34,7 +34,7 @@ public void setupTest() { _mockProducer = mock(EventProducer.class); PreProcessHooks preProcessHooks = new PreProcessHooks(); preProcessHooks.setUiEnabled(true); - _entityService = new EntityService(_aspectDao, _mockProducer, _testEntityRegistry, true, + _entityServiceImpl = new EntityServiceImpl(_aspectDao, _mockProducer, _testEntityRegistry, true, _mockUpdateIndicesService, preProcessHooks); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/timeline/TimelineServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/timeline/TimelineServiceTest.java index abd611c5a9336..b3e4b84a4962d 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/timeline/TimelineServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/timeline/TimelineServiceTest.java @@ -7,7 +7,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.data.template.RecordTemplate; import com.linkedin.metadata.entity.AspectDao; -import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.EntityServiceImpl; import com.linkedin.metadata.entity.TestEntityRegistry; import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.models.registry.ConfigEntityRegistry; @@ -61,7 +61,7 @@ abstract public class TimelineServiceTest { protected final EntityRegistry _testEntityRegistry = new MergedEntityRegistry(_snapshotEntityRegistry).apply(_configEntityRegistry); protected TimelineServiceImpl _entityTimelineService; - protected EntityService _entityService; + protected EntityServiceImpl _entityServiceImpl; protected EventProducer _mockProducer; protected UpdateIndicesService _mockUpdateIndicesService = mock(UpdateIndicesService.class); @@ -82,12 +82,12 @@ public void testGetTimeline() throws Exception { SchemaMetadata schemaMetadata = getSchemaMetadata("This is the new description for day " + i); AuditStamp daysAgo = createTestAuditStamp(i); timestamps.add(daysAgo); - _entityService.ingestAspects(entityUrn, Collections.singletonList(new Pair<>(aspectName, schemaMetadata)), + _entityServiceImpl.ingestAspects(entityUrn, Collections.singletonList(new Pair<>(aspectName, schemaMetadata)), daysAgo, getSystemMetadata(daysAgo, "run-" + i)); } Map latestAspects = - _entityService.getLatestAspectsForUrn(entityUrn, new HashSet<>(Arrays.asList(aspectName))); + _entityServiceImpl.getLatestAspectsForUrn(entityUrn, new HashSet<>(Arrays.asList(aspectName))); Set elements = new HashSet<>(); elements.add(ChangeCategory.TECHNICAL_SCHEMA); diff --git a/metadata-jobs/mae-consumer-job/src/test/java/com/linkedin/metadata/kafka/MaeConsumerApplicationTest.java b/metadata-jobs/mae-consumer-job/src/test/java/com/linkedin/metadata/kafka/MaeConsumerApplicationTest.java index 9dc813b149a31..d0190279930fe 100644 --- a/metadata-jobs/mae-consumer-job/src/test/java/com/linkedin/metadata/kafka/MaeConsumerApplicationTest.java +++ b/metadata-jobs/mae-consumer-job/src/test/java/com/linkedin/metadata/kafka/MaeConsumerApplicationTest.java @@ -15,10 +15,10 @@ public class MaeConsumerApplicationTest extends AbstractTestNGSpringContextTests { @Autowired - private EntityService mockEntityService; + private EntityService _mockEntityService; @Test public void testMaeConsumerAutoWiring() { - assertNotNull(mockEntityService); + assertNotNull(_mockEntityService); } } diff --git a/metadata-jobs/mae-consumer-job/src/test/java/com/linkedin/metadata/kafka/MaeConsumerApplicationTestConfiguration.java b/metadata-jobs/mae-consumer-job/src/test/java/com/linkedin/metadata/kafka/MaeConsumerApplicationTestConfiguration.java index 4aaee6f085201..72665ffa0b76e 100644 --- a/metadata-jobs/mae-consumer-job/src/test/java/com/linkedin/metadata/kafka/MaeConsumerApplicationTestConfiguration.java +++ b/metadata-jobs/mae-consumer-job/src/test/java/com/linkedin/metadata/kafka/MaeConsumerApplicationTestConfiguration.java @@ -3,7 +3,7 @@ import com.linkedin.entity.client.RestliEntityClient; import com.linkedin.gms.factory.auth.SystemAuthenticationFactory; import com.linkedin.metadata.dao.producer.KafkaHealthChecker; -import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.EntityServiceImpl; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.models.registry.ConfigEntityRegistry; import com.linkedin.metadata.models.registry.EntityRegistry; @@ -21,7 +21,7 @@ public class MaeConsumerApplicationTestConfiguration { private KafkaHealthChecker kafkaHealthChecker; @MockBean - private EntityService entityService; + private EntityServiceImpl _entityServiceImpl; @MockBean private RestliEntityClient restliEntityClient; diff --git a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHookTest.java b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHookTest.java index cb8b299ec75b6..030ca83131433 100644 --- a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHookTest.java +++ b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHookTest.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.kafka.hook; +import com.linkedin.metadata.config.SystemUpdateConfiguration; +import com.linkedin.metadata.config.search.ElasticSearchConfiguration; import com.linkedin.common.AuditStamp; import com.linkedin.common.InputField; import com.linkedin.common.InputFieldArray; @@ -20,8 +22,6 @@ import com.linkedin.events.metadata.ChangeType; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.metadata.Constants; -import com.linkedin.metadata.config.search.ElasticSearchConfiguration; -import com.linkedin.metadata.config.SystemUpdateConfiguration; import com.linkedin.metadata.graph.Edge; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.boot.kafka.DataHubUpgradeKafkaListener; diff --git a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTestConfiguration.java b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTestConfiguration.java index 206b15ff61cca..ef80c49ec4520 100644 --- a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTestConfiguration.java +++ b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTestConfiguration.java @@ -7,7 +7,7 @@ import com.linkedin.metadata.boot.kafka.DataHubUpgradeKafkaListener; import com.linkedin.metadata.graph.elastic.ElasticSearchGraphService; import com.linkedin.metadata.models.registry.EntityRegistry; -import com.linkedin.metadata.schema.registry.SchemaRegistryService; +import com.linkedin.metadata.registry.SchemaRegistryService; import com.linkedin.metadata.search.elasticsearch.ElasticSearchService; import com.linkedin.metadata.search.transformer.SearchDocumentTransformer; import com.linkedin.metadata.systemmetadata.SystemMetadataService; diff --git a/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTest.java b/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTest.java index 059c4ac263064..c23cf1ea3d165 100644 --- a/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTest.java +++ b/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTest.java @@ -23,13 +23,13 @@ public class MceConsumerApplicationTest extends AbstractTestNGSpringContextTests private TestRestTemplate restTemplate; @Autowired - private EntityService mockEntityService; + private EntityService _mockEntityService; @Test public void testRestliServletConfig() { RestoreIndicesResult mockResult = new RestoreIndicesResult(); mockResult.setRowsMigrated(100); - when(mockEntityService.restoreIndices(any(), any())).thenReturn(mockResult); + when(_mockEntityService.restoreIndices(any(), any())).thenReturn(mockResult); String response = this.restTemplate .postForObject("/gms/aspects?action=restoreIndices", "{\"urn\":\"\"}", String.class); diff --git a/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTestConfiguration.java b/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTestConfiguration.java index 747a4ab312c7a..2d09cf2043575 100644 --- a/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTestConfiguration.java +++ b/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTestConfiguration.java @@ -33,7 +33,7 @@ public class MceConsumerApplicationTestConfiguration { public KafkaHealthChecker kafkaHealthChecker; @MockBean - public EntityService entityService; + public EntityService _entityService; @Bean("restliEntityClient") @Primary diff --git a/metadata-service/configuration/build.gradle b/metadata-service/configuration/build.gradle new file mode 100644 index 0000000000000..8623e53d2554a --- /dev/null +++ b/metadata-service/configuration/build.gradle @@ -0,0 +1,14 @@ +plugins { + id 'java' +} + +dependencies { + compile externalDependency.jacksonDataBind + + implementation externalDependency.slf4jApi + implementation externalDependency.springCore + + compileOnly externalDependency.lombok + + annotationProcessor externalDependency.lombok +} \ No newline at end of file diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/AssetsConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/AssetsConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/AssetsConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/AssetsConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/AuthPluginConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/AuthPluginConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/AuthPluginConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/AuthPluginConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/DataHubConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/DataHubConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/DataHubConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/DataHubConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/EntityProfileConfig.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/EntityProfileConfig.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/EntityProfileConfig.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/EntityProfileConfig.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/EntityRegistryPluginConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/EntityRegistryPluginConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/EntityRegistryPluginConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/EntityRegistryPluginConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/IngestionConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/IngestionConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/IngestionConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/IngestionConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/PluginConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/PluginConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/PluginConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/PluginConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/PreProcessHooks.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/PreProcessHooks.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/PreProcessHooks.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/PreProcessHooks.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/QueriesTabConfig.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/QueriesTabConfig.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/QueriesTabConfig.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/QueriesTabConfig.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/RetentionPluginConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/RetentionPluginConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/RetentionPluginConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/RetentionPluginConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/SystemUpdateConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/SystemUpdateConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/SystemUpdateConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/SystemUpdateConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/TestsConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/TestsConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/TestsConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/TestsConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/ViewsConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/ViewsConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/ViewsConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/ViewsConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/VisualConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/VisualConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/VisualConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/VisualConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/cache/CacheConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/cache/CacheConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/cache/CacheConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/cache/CacheConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/cache/EntityDocCountCacheConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/cache/EntityDocCountCacheConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/cache/EntityDocCountCacheConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/cache/EntityDocCountCacheConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/cache/HomepageCacheConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/cache/HomepageCacheConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/cache/HomepageCacheConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/cache/HomepageCacheConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/cache/PrimaryCacheConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/cache/PrimaryCacheConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/cache/PrimaryCacheConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/cache/PrimaryCacheConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/cache/SearchCacheConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/cache/SearchCacheConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/cache/SearchCacheConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/cache/SearchCacheConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/cache/SearchLineageCacheConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/cache/SearchLineageCacheConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/cache/SearchLineageCacheConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/cache/SearchLineageCacheConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/kafka/KafkaConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/KafkaConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/kafka/KafkaConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/KafkaConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/kafka/ListenerConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/ListenerConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/kafka/ListenerConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/ListenerConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/kafka/ProducerConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/ProducerConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/kafka/ProducerConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/ProducerConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/kafka/SchemaRegistryConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/SchemaRegistryConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/kafka/SchemaRegistryConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/SchemaRegistryConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/search/BuildIndicesConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/BuildIndicesConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/search/BuildIndicesConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/BuildIndicesConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/search/CustomConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/CustomConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/search/CustomConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/CustomConfiguration.java index d1a1ddc5d4226..7a0292c2adec1 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/config/search/CustomConfiguration.java +++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/CustomConfiguration.java @@ -1,7 +1,7 @@ package com.linkedin.metadata.config.search; -import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; +import com.fasterxml.jackson.databind.ObjectMapper; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.springframework.core.io.ClassPathResource; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/search/ElasticSearchConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/ElasticSearchConfiguration.java similarity index 75% rename from metadata-io/src/main/java/com/linkedin/metadata/config/search/ElasticSearchConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/ElasticSearchConfiguration.java index 7525ea7202a99..30679bbaab9ce 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/config/search/ElasticSearchConfiguration.java +++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/ElasticSearchConfiguration.java @@ -1,11 +1,9 @@ package com.linkedin.metadata.config.search; import lombok.Data; -import org.springframework.context.annotation.Configuration; @Data -@Configuration public class ElasticSearchConfiguration { private BuildIndicesConfiguration buildIndices; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/search/ExactMatchConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/ExactMatchConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/search/ExactMatchConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/ExactMatchConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/search/GraphQueryConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/GraphQueryConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/search/GraphQueryConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/GraphQueryConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/search/PartialConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/PartialConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/search/PartialConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/PartialConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/search/SearchConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/SearchConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/search/SearchConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/SearchConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/search/custom/BoolQueryConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/custom/BoolQueryConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/search/custom/BoolQueryConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/custom/BoolQueryConfiguration.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/search/custom/CustomSearchConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/custom/CustomSearchConfiguration.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/config/search/custom/CustomSearchConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/custom/CustomSearchConfiguration.java diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/custom/QueryConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/custom/QueryConfiguration.java new file mode 100644 index 0000000000000..cd4364a64a0c5 --- /dev/null +++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/custom/QueryConfiguration.java @@ -0,0 +1,35 @@ +package com.linkedin.metadata.config.search.custom; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; + + +@Slf4j +@Builder(toBuilder = true) +@Getter +@ToString +@EqualsAndHashCode +@JsonDeserialize(builder = QueryConfiguration.QueryConfigurationBuilder.class) +public class QueryConfiguration { + + private String queryRegex; + @Builder.Default + private boolean simpleQuery = true; + @Builder.Default + private boolean exactMatchQuery = true; + @Builder.Default + private boolean prefixMatchQuery = true; + private BoolQueryConfiguration boolQuery; + private Map functionScore; + + @JsonPOJOBuilder(withPrefix = "") + public static class QueryConfigurationBuilder { + } +} diff --git a/metadata-io/src/main/java/com/linkedin/metadata/telemetry/TelemetryConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/telemetry/TelemetryConfiguration.java similarity index 92% rename from metadata-io/src/main/java/com/linkedin/metadata/telemetry/TelemetryConfiguration.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/config/telemetry/TelemetryConfiguration.java index 16fdcdb6f3fbb..3821cbbed83e8 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/telemetry/TelemetryConfiguration.java +++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/telemetry/TelemetryConfiguration.java @@ -1,4 +1,4 @@ -package com.linkedin.metadata.telemetry; +package com.linkedin.metadata.config.telemetry; import lombok.Data; /** diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/BatchWriteOperationsOptions.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/timeseries/BatchWriteOperationsOptions.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/timeseries/BatchWriteOperationsOptions.java rename to metadata-service/configuration/src/main/java/com/linkedin/metadata/timeseries/BatchWriteOperationsOptions.java diff --git a/metadata-service/factories/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml similarity index 100% rename from metadata-service/factories/src/main/resources/application.yml rename to metadata-service/configuration/src/main/resources/application.yml diff --git a/metadata-service/factories/build.gradle b/metadata-service/factories/build.gradle index e416580053120..796b6ee436b78 100644 --- a/metadata-service/factories/build.gradle +++ b/metadata-service/factories/build.gradle @@ -7,6 +7,7 @@ dependencies { compile project(':metadata-service:auth-impl') compile project(':metadata-service:auth-config') compile project(':metadata-service:plugin') + compile project(':metadata-service:configuration') compile project(':datahub-graphql-core') compile project(':metadata-service:restli-servlet-impl') compile project(':metadata-dao-impl:kafka-producer') diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubTokenServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubTokenServiceFactory.java index 20f2e3829f8c1..fc010a1aa2cae 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubTokenServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubTokenServiceFactory.java @@ -36,7 +36,7 @@ public class DataHubTokenServiceFactory { */ @Autowired @Qualifier("entityService") - private EntityService entityService; + private EntityService _entityService; @Bean(name = "dataHubTokenService") @Scope("singleton") @@ -46,7 +46,7 @@ protected StatefulTokenService getInstance() { this.signingKey, this.signingAlgorithm, this.issuer, - this.entityService, + this._entityService, this.saltingKey ); } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java index ef4af64ea0493..e07630111a567 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java @@ -2,18 +2,18 @@ import com.datahub.authentication.AuthenticationConfiguration; import com.datahub.authorization.AuthorizationConfiguration; -import com.linkedin.datahub.graphql.featureflags.FeatureFlags; -import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; -import com.linkedin.metadata.config.cache.CacheConfiguration; import com.linkedin.metadata.config.DataHubConfiguration; -import com.linkedin.metadata.config.search.ElasticSearchConfiguration; import com.linkedin.metadata.config.IngestionConfiguration; import com.linkedin.metadata.config.SystemUpdateConfiguration; import com.linkedin.metadata.config.TestsConfiguration; import com.linkedin.metadata.config.ViewsConfiguration; import com.linkedin.metadata.config.VisualConfiguration; +import com.linkedin.metadata.config.cache.CacheConfiguration; import com.linkedin.metadata.config.kafka.KafkaConfiguration; -import com.linkedin.metadata.telemetry.TelemetryConfiguration; +import com.linkedin.metadata.config.search.ElasticSearchConfiguration; +import com.linkedin.datahub.graphql.featureflags.FeatureFlags; +import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; +import com.linkedin.metadata.config.telemetry.TelemetryConfiguration; import lombok.Data; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityServiceFactory.java index 6bb86b01604c1..b8edd6918dc6c 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityServiceFactory.java @@ -7,6 +7,7 @@ import com.linkedin.metadata.dao.producer.KafkaHealthChecker; import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.EntityServiceImpl; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.service.UpdateIndicesService; import com.linkedin.mxe.TopicConvention; @@ -38,7 +39,7 @@ protected EntityService createInstance( final KafkaEventProducer eventProducer = new KafkaEventProducer(producer, convention, kafkaHealthChecker); FeatureFlags featureFlags = configurationProvider.getFeatureFlags(); - return new EntityService(aspectDao, eventProducer, entityRegistry, + return new EntityServiceImpl(aspectDao, eventProducer, entityRegistry, featureFlags.isAlwaysEmitChangeLog(), updateIndicesService, featureFlags.getPreProcessHooks()); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/DataHubKafkaProducerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/DataHubKafkaProducerFactory.java index 251d674909d9f..e58661b357e6a 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/DataHubKafkaProducerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/DataHubKafkaProducerFactory.java @@ -1,11 +1,11 @@ package com.linkedin.gms.factory.kafka; +import com.linkedin.metadata.config.kafka.KafkaConfiguration; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.gms.factory.kafka.schemaregistry.AwsGlueSchemaRegistryFactory; import com.linkedin.gms.factory.kafka.schemaregistry.InternalSchemaRegistryFactory; import com.linkedin.gms.factory.kafka.schemaregistry.KafkaSchemaRegistryFactory; import com.linkedin.gms.factory.kafka.schemaregistry.SchemaRegistryConfig; -import com.linkedin.metadata.config.kafka.KafkaConfiguration; import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; import java.util.Arrays; import java.util.Map; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java index 5362ad0b11648..ba18be6834d14 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java @@ -1,11 +1,11 @@ package com.linkedin.gms.factory.kafka; +import com.linkedin.metadata.config.kafka.KafkaConfiguration; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.gms.factory.kafka.schemaregistry.AwsGlueSchemaRegistryFactory; import com.linkedin.gms.factory.kafka.schemaregistry.InternalSchemaRegistryFactory; import com.linkedin.gms.factory.kafka.schemaregistry.KafkaSchemaRegistryFactory; import com.linkedin.gms.factory.kafka.schemaregistry.SchemaRegistryConfig; -import com.linkedin.metadata.config.kafka.KafkaConfiguration; import java.time.Duration; import java.util.Arrays; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactory.java index 5c402548b7cbd..05ebfdddf8b80 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactory.java @@ -1,7 +1,7 @@ package com.linkedin.gms.factory.kafka; -import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.metadata.config.kafka.KafkaConfiguration; +import com.linkedin.gms.factory.config.ConfigurationProvider; import java.time.Duration; import java.util.Arrays; import lombok.extern.slf4j.Slf4j; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/DUHESchemaRegistryFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/DUHESchemaRegistryFactory.java index e282cc3861b41..aeef166a077c7 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/DUHESchemaRegistryFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/DUHESchemaRegistryFactory.java @@ -1,9 +1,9 @@ package com.linkedin.gms.factory.kafka.schemaregistry; +import com.linkedin.metadata.config.kafka.KafkaConfiguration; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.metadata.boot.kafka.MockDUHEDeserializer; import com.linkedin.metadata.boot.kafka.MockDUHESerializer; -import com.linkedin.metadata.config.kafka.KafkaConfiguration; import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/InternalSchemaRegistryFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/InternalSchemaRegistryFactory.java index b103b84307611..217dc15bbc3e8 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/InternalSchemaRegistryFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/InternalSchemaRegistryFactory.java @@ -1,10 +1,10 @@ package com.linkedin.gms.factory.kafka.schemaregistry; +import com.linkedin.metadata.config.kafka.KafkaConfiguration; import com.linkedin.gms.factory.common.TopicConventionFactory; import com.linkedin.gms.factory.config.ConfigurationProvider; -import com.linkedin.metadata.config.kafka.KafkaConfiguration; -import com.linkedin.metadata.schema.registry.SchemaRegistryService; -import com.linkedin.metadata.schema.registry.SchemaRegistryServiceImpl; +import com.linkedin.metadata.registry.SchemaRegistryService; +import com.linkedin.metadata.registry.SchemaRegistryServiceImpl; import com.linkedin.mxe.TopicConvention; import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig; import io.confluent.kafka.serializers.KafkaAvroDeserializer; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyEditedCandidateSourceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyEditedCandidateSourceFactory.java index 836824391bcf7..58584a4d957de 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyEditedCandidateSourceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyEditedCandidateSourceFactory.java @@ -28,11 +28,11 @@ public class RecentlyEditedCandidateSourceFactory { @Autowired @Qualifier("entityService") - private EntityService entityService; + private EntityService _entityService; @Bean(name = "recentlyEditedCandidateSource") @Nonnull protected RecentlyEditedSource getInstance() { - return new RecentlyEditedSource(searchClient, indexConvention, entityService); + return new RecentlyEditedSource(searchClient, indexConvention, _entityService); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchServiceFactory.java index 0aa1c1a9cfd23..03dd2d072b4a0 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchServiceFactory.java @@ -1,13 +1,13 @@ package com.linkedin.gms.factory.search; +import com.linkedin.metadata.config.search.ElasticSearchConfiguration; +import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.gms.factory.entityregistry.EntityRegistryFactory; import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; -import com.linkedin.metadata.config.search.ElasticSearchConfiguration; -import com.linkedin.metadata.config.search.SearchConfiguration; -import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.elasticsearch.ElasticSearchService; import com.linkedin.metadata.search.elasticsearch.indexbuilder.EntityIndexBuilders; diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java index 627c5934140a8..ea9ac57778550 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java @@ -26,6 +26,9 @@ import javax.annotation.Nonnull; import java.util.Set; +import static com.linkedin.metadata.Constants.*; + + @Slf4j public class BackfillBrowsePathsV2Step extends UpgradeStep { @@ -133,7 +136,7 @@ private void ingestBrowsePathsV2(Urn urn, AuditStamp auditStamp) throws Exceptio proposal.setEntityType(urn.getEntityType()); proposal.setAspectName(Constants.BROWSE_PATHS_V2_ASPECT_NAME); proposal.setChangeType(ChangeType.UPSERT); - proposal.setSystemMetadata(new SystemMetadata().setRunId(EntityService.DEFAULT_RUN_ID).setLastObserved(System.currentTimeMillis())); + proposal.setSystemMetadata(new SystemMetadata().setRunId(DEFAULT_RUN_ID).setLastObserved(System.currentTimeMillis())); proposal.setAspect(GenericRecordUtils.serializeAspect(browsePathsV2)); _entityService.ingestProposal( proposal, diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStep.java index b990400b38491..7fcafa24d7b45 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStep.java @@ -19,6 +19,8 @@ import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; +import static com.linkedin.metadata.Constants.*; + /** * This is an opt-in optional upgrade step to migrate your browse paths to the new truncated form. @@ -124,7 +126,7 @@ private void migrateBrowsePath(Urn urn, AuditStamp auditStamp) throws Exception proposal.setEntityType(urn.getEntityType()); proposal.setAspectName(Constants.BROWSE_PATHS_ASPECT_NAME); proposal.setChangeType(ChangeType.UPSERT); - proposal.setSystemMetadata(new SystemMetadata().setRunId(EntityService.DEFAULT_RUN_ID).setLastObserved(System.currentTimeMillis())); + proposal.setSystemMetadata(new SystemMetadata().setRunId(DEFAULT_RUN_ID).setLastObserved(System.currentTimeMillis())); proposal.setAspect(GenericRecordUtils.serializeAspect(newPaths)); _entityService.ingestProposal( proposal, diff --git a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStepTest.java b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStepTest.java index fa351d7539e02..82f3a82c135ce 100644 --- a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStepTest.java +++ b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStepTest.java @@ -127,8 +127,10 @@ private void mockDBWithWorkToDo( AspectMigrationsDao migrationsDao, int countOfCorpUserEntities, int countOfChartEntities) { - List corpUserUrns = insertMockEntities(countOfCorpUserEntities, "corpuser", "urn:li:corpuser:test%d", entityRegistry, entityService); - List charUrns = insertMockEntities(countOfChartEntities, "chart", "urn:li:chart:(looker,test%d)", entityRegistry, entityService); + List corpUserUrns = insertMockEntities(countOfCorpUserEntities, "corpuser", "urn:li:corpuser:test%d", entityRegistry, + entityService); + List charUrns = insertMockEntities(countOfChartEntities, "chart", "urn:li:chart:(looker,test%d)", entityRegistry, + entityService); List allUrnsInDB = Stream.concat(corpUserUrns.stream(), charUrns.stream()).map(Urn::toString).collect(Collectors.toList()); when(migrationsDao.checkIfAspectExists(DATA_PLATFORM_INSTANCE_ASPECT_NAME)).thenReturn(false); when(migrationsDao.countEntities()).thenReturn((long) allUrnsInDB.size()); diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java index 7ce41924fe92d..52842d2f32b2a 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java @@ -19,6 +19,7 @@ import com.linkedin.entity.Aspect; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.IngestProposalResult; import com.linkedin.metadata.entity.RollbackRunResult; import com.linkedin.metadata.entity.validation.ValidationException; import com.linkedin.metadata.models.EntitySpec; @@ -265,7 +266,7 @@ public static Pair ingestProposal(com.linkedin.mxe.MetadataChan log.info("Proposal: {}", serviceProposal); Throwable exceptionally = null; try { - EntityService.IngestProposalResult proposalResult = entityService.ingestProposal(serviceProposal, auditStamp, false); + IngestProposalResult proposalResult = entityService.ingestProposal(serviceProposal, auditStamp, false); Urn urn = proposalResult.getUrn(); additionalChanges.forEach(proposal -> entityService.ingestProposal(proposal, auditStamp, false)); return new Pair<>(urn.toString(), proposalResult.isDidUpdate()); diff --git a/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java b/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java index 2dff636de2f27..80cc80067fba2 100644 --- a/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java +++ b/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java @@ -6,8 +6,8 @@ import com.datahub.authentication.AuthenticationContext; import com.datahub.authorization.AuthorizationResult; import com.datahub.authorization.AuthorizerChain; -import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.metadata.config.PreProcessHooks; +import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.models.registry.EntityRegistry; diff --git a/metadata-service/openapi-servlet/src/test/java/mock/MockEntityService.java b/metadata-service/openapi-servlet/src/test/java/mock/MockEntityService.java index 9838b00abd191..cf7985a7c26a9 100644 --- a/metadata-service/openapi-servlet/src/test/java/mock/MockEntityService.java +++ b/metadata-service/openapi-servlet/src/test/java/mock/MockEntityService.java @@ -1,5 +1,6 @@ package mock; +import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.common.AuditStamp; import com.linkedin.common.GlobalTags; import com.linkedin.common.GlossaryTermAssociation; @@ -19,11 +20,11 @@ import com.linkedin.entity.AspectType; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.metadata.aspect.VersionedAspect; -import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.entity.AspectDao; -import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.EntityServiceImpl; import com.linkedin.metadata.entity.ListResult; import com.linkedin.metadata.entity.RollbackRunResult; +import com.linkedin.metadata.entity.UpdateAspectResult; import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.registry.EntityRegistry; @@ -56,7 +57,7 @@ import static entities.EntitiesControllerTest.*; -public class MockEntityService extends EntityService { +public class MockEntityService extends EntityServiceImpl { public MockEntityService(@Nonnull AspectDao aspectDao, @Nonnull EventProducer producer, @Nonnull EntityRegistry entityRegistry, @Nonnull UpdateIndicesService updateIndicesService, PreProcessHooks preProcessHooks) { super(aspectDao, producer, entityRegistry, true, updateIndicesService, preProcessHooks); diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java index 0004c82b43a6f..5c06dbfc8b25a 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java @@ -16,6 +16,7 @@ import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.metadata.entity.AspectUtils; import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.IngestProposalResult; import com.linkedin.metadata.entity.validation.ValidationException; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.query.filter.Filter; @@ -199,7 +200,7 @@ public Task ingestProposal( return RestliUtil.toTask(() -> { log.debug("Proposal: {}", metadataChangeProposal); try { - EntityService.IngestProposalResult result = _entityService.ingestProposal(metadataChangeProposal, auditStamp, asyncBool); + IngestProposalResult result = _entityService.ingestProposal(metadataChangeProposal, auditStamp, asyncBool); Urn responseUrn = result.getUrn(); if (!asyncBool) { diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/BatchIngestionRunResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/BatchIngestionRunResource.java index 674609c7f67df..3ff22fb767676 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/BatchIngestionRunResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/BatchIngestionRunResource.java @@ -110,7 +110,7 @@ public Task rollback(@ActionParam("runId") @Nonnull String run } try { return RestliUtil.toTask(() -> { - if (runId.equals(EntityService.DEFAULT_RUN_ID)) { + if (runId.equals(DEFAULT_RUN_ID)) { throw new IllegalArgumentException(String.format( "%s is a default run-id provided for non labeled ingestion runs. You cannot delete using this reserved run-id", runId)); diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java index 3e90f4dff2ccf..b8fd785eaad0f 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java @@ -428,7 +428,8 @@ public Task scrollAcrossLineage(@ActionParam(PARAM_URN) @No final SearchFlags finalFlags = searchFlags != null ? searchFlags : new SearchFlags().setSkipCache(true); return RestliUtil.toTask(() -> validateLineageScrollResult( _lineageSearchService.scrollAcrossLineage(urn, LineageDirection.valueOf(direction), entityList, input, maxHops, - filter, sortCriterion, scrollId, keepAlive, count, startTimeMillis, endTimeMillis, finalFlags), _entityService), + filter, sortCriterion, scrollId, keepAlive, count, startTimeMillis, endTimeMillis, finalFlags), + _entityService), "scrollAcrossLineage"); } diff --git a/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/entity/AspectResourceTest.java b/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/entity/AspectResourceTest.java index 0d3642b91758a..c6d36a6e29f10 100644 --- a/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/entity/AspectResourceTest.java +++ b/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/entity/AspectResourceTest.java @@ -14,6 +14,8 @@ import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.EntityServiceImpl; +import com.linkedin.metadata.entity.UpdateAspectResult; import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.registry.EntityRegistry; @@ -48,7 +50,7 @@ public void setup() { _entityRegistry = new MockEntityRegistry(); _updateIndicesService = mock(UpdateIndicesService.class); _preProcessHooks = mock(PreProcessHooks.class); - _entityService = new EntityService(_aspectDao, _producer, _entityRegistry, false, _updateIndicesService, _preProcessHooks); + _entityService = new EntityServiceImpl(_aspectDao, _producer, _entityRegistry, false, _updateIndicesService, _preProcessHooks); _authorizer = mock(Authorizer.class); _aspectResource.setAuthorizer(_authorizer); _aspectResource.setEntityService(_entityService); @@ -77,7 +79,7 @@ public void testAsyncDefaultAspects() throws URISyntaxException { reset(_producer, _aspectDao); when(_aspectDao.runInTransactionWithRetry(any(), anyInt())) - .thenReturn(new EntityService.UpdateAspectResult(urn, null, properties, null, null, null, null, 0)); + .thenReturn(new UpdateAspectResult(urn, null, properties, null, null, null, null, 0)); _aspectResource.ingestProposal(mcp, "false"); verify(_producer, times(5)).produceMetadataChangeLog(eq(urn), any(AspectSpec.class), any(MetadataChangeLog.class)); verifyNoMoreInteractions(_producer); diff --git a/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/SchemaRegistryController.java b/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/SchemaRegistryController.java index 3c90336a142ee..0cf57361e58f8 100644 --- a/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/SchemaRegistryController.java +++ b/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/SchemaRegistryController.java @@ -2,7 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.gms.factory.kafka.schemaregistry.InternalSchemaRegistryFactory; -import com.linkedin.metadata.schema.registry.SchemaRegistryService; +import com.linkedin.metadata.registry.SchemaRegistryService; import io.datahubproject.schema_registry.openapi.generated.CompatibilityCheckResponse; import io.datahubproject.schema_registry.openapi.generated.Config; import io.datahubproject.schema_registry.openapi.generated.ConfigUpdateRequest; diff --git a/metadata-service/services/README.md b/metadata-service/services/README.md new file mode 100644 index 0000000000000..510f8756a7f5e --- /dev/null +++ b/metadata-service/services/README.md @@ -0,0 +1,5 @@ +# Service Layer + +Module to abstract away business logic from implementation specific libraries to make them lighter weight from a +dependency perspective. Service classes should be here unless they require direct usage of implementation specific libraries +(i.e. ElasticSearch, Ebean, Neo4J, etc.). \ No newline at end of file diff --git a/metadata-service/services/build.gradle b/metadata-service/services/build.gradle new file mode 100644 index 0000000000000..adc7b7bf09d99 --- /dev/null +++ b/metadata-service/services/build.gradle @@ -0,0 +1,73 @@ +apply plugin: 'java' +apply plugin: 'org.hidetake.swagger.generator' + +configurations { + enhance +} + +dependencies { + implementation externalDependency.jsonPatch + compile project(':entity-registry') + compile project(':metadata-utils') + compile project(':metadata-events:mxe-avro-1.7') + compile project(':metadata-events:mxe-registration') + compile project(':metadata-events:mxe-utils-avro-1.7') + compile project(':metadata-models') + compile project(':metadata-service:restli-client') + compile project(':metadata-service:configuration') + + implementation externalDependency.slf4jApi + implementation externalDependency.swaggerAnnotations + runtime externalDependency.logbackClassic + compileOnly externalDependency.lombok + implementation externalDependency.commonsCollections + compile externalDependency.javatuples + compile externalDependency.javaxValidation + compile externalDependency.opentelemetryAnnotations + + annotationProcessor externalDependency.lombok + + testCompile externalDependency.testng + testCompile externalDependency.junit + testCompile externalDependency.mockito + testCompile externalDependency.mockitoInline + testCompileOnly externalDependency.lombok + testCompile project(':test-models') + testImplementation project(':datahub-graphql-core') + // logback >=1.3 required due to `testcontainers` only + testImplementation 'ch.qos.logback:logback-classic:1.4.7' + + testAnnotationProcessor externalDependency.lombok + + constraints { + implementation(externalDependency.log4jCore) { + because("previous versions are vulnerable to CVE-2021-45105") + } + implementation(externalDependency.log4jApi) { + because("previous versions are vulnerable to CVE-2021-45105") + } + implementation(externalDependency.commonsText) { + because("previous versions are vulnerable to CVE-2022-42889") + } + implementation(externalDependency.snakeYaml) { + because("previous versions are vulnerable to CVE-2022-25857") + } + implementation(externalDependency.woodstoxCore) { + because("previous versions are vulnerable to CVE-2022-40151-2") + } + implementation(externalDependency.jettison) { + because("previous versions are vulnerable") + } + } +} + +test { + // https://docs.gradle.org/current/userguide/performance.html + maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1 + testLogging.showStandardStreams = true + testLogging.exceptionFormat = 'full' +} + +tasks.withType(Test) { + enableAssertions = false +} diff --git a/metadata-io/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventConstants.java b/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventConstants.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventConstants.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventConstants.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java b/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectUtils.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/AspectUtils.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/entity/AspectUtils.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/entity/AspectUtils.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/DeleteEntityService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/DeleteEntityService.java similarity index 99% rename from metadata-io/src/main/java/com/linkedin/metadata/entity/DeleteEntityService.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/entity/DeleteEntityService.java index bf077dfe0eb21..35b7bc4589b32 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/DeleteEntityService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/DeleteEntityService.java @@ -270,7 +270,7 @@ private void updateAspect(Urn urn, String aspectName, RecordTemplate prevAspect, proposal.setAspect(GenericRecordUtils.serializeAspect(newAspect)); final AuditStamp auditStamp = new AuditStamp().setActor(UrnUtils.getUrn(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis()); - final EntityService.IngestProposalResult ingestProposalResult = _entityService.ingestProposal(proposal, auditStamp, false); + final IngestProposalResult ingestProposalResult = _entityService.ingestProposal(proposal, auditStamp, false); if (!ingestProposalResult.isDidUpdate()) { log.error("Failed to ingest aspect with references removed. Before {}, after: {}, please check MCP processor" diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/DeleteEntityUtils.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/DeleteEntityUtils.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/entity/DeleteEntityUtils.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/entity/DeleteEntityUtils.java diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java new file mode 100644 index 0000000000000..25edff740037e --- /dev/null +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java @@ -0,0 +1,311 @@ +package com.linkedin.metadata.entity; + +import com.linkedin.common.AuditStamp; +import com.linkedin.common.BrowsePaths; +import com.linkedin.common.BrowsePathsV2; +import com.linkedin.common.VersionedUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.entity.Entity; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.aspect.VersionedAspect; +import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; +import com.linkedin.metadata.entity.restoreindices.RestoreIndicesResult; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.query.ListUrnsResult; +import com.linkedin.metadata.run.AspectRowSummary; +import com.linkedin.metadata.snapshot.Snapshot; +import com.linkedin.mxe.MetadataAuditOperation; +import com.linkedin.mxe.MetadataChangeLog; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.mxe.SystemMetadata; +import com.linkedin.util.Pair; +import java.net.URISyntaxException; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Consumer; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + + +public interface EntityService { + + /** + * Retrieves the latest aspects corresponding to a batch of {@link Urn}s based on a provided + * set of aspect names. + * + * @param urns set of urns to fetch aspects for + * @param aspectNames aspects to fetch for each urn in urns set + * @return a map of provided {@link Urn} to a List containing the requested aspects. + */ + Map> getLatestAspects( + @Nonnull final Set urns, + @Nonnull final Set aspectNames); + + Map getLatestAspectsForUrn(@Nonnull final Urn urn, @Nonnull final Set aspectNames); + + /** + * Retrieves an aspect having a specific {@link Urn}, name, & version. + * + * Note that once we drop support for legacy aspect-specific resources, + * we should make this a protected method. Only visible for backwards compatibility. + * + * @param urn an urn associated with the requested aspect + * @param aspectName name of the aspect requested + * @param version specific version of the aspect being requests + * @return the {@link RecordTemplate} representation of the requested aspect object, or null if one cannot be found + */ + RecordTemplate getAspect(@Nonnull final Urn urn, @Nonnull final String aspectName, @Nonnull long version); + + /** + * Retrieves the latest aspects for the given urn as dynamic aspect objects + * (Without having to define union objects) + * + * @param entityName name of the entity to fetch + * @param urn urn of entity to fetch + * @param aspectNames set of aspects to fetch + * @return a map of {@link Urn} to {@link Entity} object + */ + EntityResponse getEntityV2( + @Nonnull final String entityName, + @Nonnull final Urn urn, + @Nonnull final Set aspectNames) throws URISyntaxException; + + /** + * Retrieves the latest aspects for the given set of urns as dynamic aspect objects + * (Without having to define union objects) + * + * @param entityName name of the entity to fetch + * @param urns set of urns to fetch + * @param aspectNames set of aspects to fetch + * @return a map of {@link Urn} to {@link Entity} object + */ + Map getEntitiesV2( + @Nonnull final String entityName, + @Nonnull final Set urns, + @Nonnull final Set aspectNames) throws URISyntaxException; + + /** + * Retrieves the aspects for the given set of urns and versions as dynamic aspect objects + * (Without having to define union objects) + * + * @param versionedUrns set of urns to fetch with versions of aspects specified in a specialized string + * @param aspectNames set of aspects to fetch + * @return a map of {@link Urn} to {@link Entity} object + */ + Map getEntitiesVersionedV2( + @Nonnull final Set versionedUrns, + @Nonnull final Set aspectNames) throws URISyntaxException; + + /** + * Retrieves the latest aspects for the given set of urns as a list of enveloped aspects + * + * @param entityName name of the entity to fetch + * @param urns set of urns to fetch + * @param aspectNames set of aspects to fetch + * @return a map of {@link Urn} to {@link EnvelopedAspect} object + */ + Map> getLatestEnvelopedAspects( + // TODO: entityName is unused, can we remove this as a param? + @Nonnull String entityName, + @Nonnull Set urns, + @Nonnull Set aspectNames) throws URISyntaxException; + + /** + * Retrieves the latest aspects for the given set of urns as a list of enveloped aspects + * + * @param versionedUrns set of urns to fetch with versions of aspects specified in a specialized string + * @param aspectNames set of aspects to fetch + * @return a map of {@link Urn} to {@link EnvelopedAspect} object + */ + Map> getVersionedEnvelopedAspects( + @Nonnull Set versionedUrns, + @Nonnull Set aspectNames) throws URISyntaxException; + + /** + * Retrieves the latest aspect for the given urn as a list of enveloped aspects + * + * @param entityName name of the entity to fetch + * @param urn urn to fetch + * @param aspectName name of the aspect to fetch + * @return {@link EnvelopedAspect} object, or null if one cannot be found + */ + EnvelopedAspect getLatestEnvelopedAspect( + @Nonnull final String entityName, + @Nonnull final Urn urn, + @Nonnull final String aspectName) throws Exception; + + /** + * Retrieves the specific version of the aspect for the given urn + * + * @param entityName name of the entity to fetch + * @param urn urn to fetch + * @param aspectName name of the aspect to fetch + * @param version version to fetch + * @return {@link EnvelopedAspect} object, or null if one cannot be found + */ + EnvelopedAspect getEnvelopedAspect( + // TODO: entityName is only used for a debug statement, can we remove this as a param? + String entityName, + @Nonnull Urn urn, + @Nonnull String aspectName, + long version) throws Exception; + + @Deprecated + VersionedAspect getVersionedAspect(@Nonnull Urn urn, @Nonnull String aspectName, long version); + + ListResult listLatestAspects( + @Nonnull final String entityName, + @Nonnull final String aspectName, + final int start, + final int count); + + void ingestAspects(@Nonnull final Urn urn, @Nonnull List> aspectRecordsToIngest, + @Nonnull final AuditStamp auditStamp, @Nullable SystemMetadata systemMetadata); + + /** + * Ingests (inserts) a new version of an entity aspect & emits a {@link com.linkedin.mxe.MetadataAuditEvent}. + * + * Note that in general, this should not be used externally. It is currently serving upgrade scripts and + * is as such public. + * + * @param urn an urn associated with the new aspect + * @param aspectName name of the aspect being inserted + * @param newValue value of the aspect being inserted + * @param auditStamp an {@link AuditStamp} containing metadata about the writer & current time + * @param systemMetadata + * @return the {@link RecordTemplate} representation of the written aspect object + */ + RecordTemplate ingestAspect(@Nonnull final Urn urn, @Nonnull final String aspectName, + @Nonnull final RecordTemplate newValue, @Nonnull final AuditStamp auditStamp, @Nullable SystemMetadata systemMetadata); + + /** + * Ingests (inserts) a new version of an entity aspect & emits a {@link com.linkedin.mxe.MetadataAuditEvent}. + * + * This method runs a read -> write atomically in a single transaction, this is to prevent multiple IDs from being created. + * + * Note that in general, this should not be used externally. It is currently serving upgrade scripts and + * is as such public. + * + * @param urn an urn associated with the new aspect + * @param aspectName name of the aspect being inserted + * @param newValue value of the aspect being inserted + * @param auditStamp an {@link AuditStamp} containing metadata about the writer & current time + * @param systemMetadata + * @return the {@link RecordTemplate} representation of the written aspect object + */ + RecordTemplate ingestAspectIfNotPresent(@Nonnull Urn urn, @Nonnull String aspectName, + @Nonnull RecordTemplate newValue, @Nonnull AuditStamp auditStamp, @Nullable SystemMetadata systemMetadata); + + // TODO: Why not in RetentionService? + String batchApplyRetention(Integer start, Integer count, Integer attemptWithVersion, String aspectName, + String urn); + + Integer getCountAspect(@Nonnull String aspectName, @Nullable String urnLike); + + // TODO: Extract this to a different service, doesn't need to be here + RestoreIndicesResult restoreIndices(@Nonnull RestoreIndicesArgs args, @Nonnull Consumer logger); + + @Deprecated + RecordTemplate updateAspect( + @Nonnull final Urn urn, + @Nonnull final String entityName, + @Nonnull final String aspectName, + @Nonnull final AspectSpec aspectSpec, + @Nonnull final RecordTemplate newValue, + @Nonnull final AuditStamp auditStamp, + @Nonnull final long version, + @Nonnull final boolean emitMae); + + ListUrnsResult listUrns(@Nonnull final String entityName, final int start, final int count); + + @Deprecated + Entity getEntity(@Nonnull final Urn urn, @Nonnull final Set aspectNames); + + @Deprecated + Map getEntities(@Nonnull final Set urns, @Nonnull Set aspectNames); + + @Deprecated + void produceMetadataAuditEvent(@Nonnull final Urn urn, @Nonnull final String aspectName, + @Nullable final RecordTemplate oldAspectValue, @Nullable final RecordTemplate newAspectValue, + @Nullable final SystemMetadata oldSystemMetadata, @Nullable final SystemMetadata newSystemMetadata, + @Nullable final MetadataAuditOperation operation); + + @Deprecated + void produceMetadataAuditEventForKey(@Nonnull final Urn urn, + @Nullable final SystemMetadata newSystemMetadata); + + void produceMetadataChangeLog(@Nonnull final Urn urn, AspectSpec aspectSpec, + @Nonnull final MetadataChangeLog metadataChangeLog); + + void produceMetadataChangeLog(@Nonnull final Urn urn, @Nonnull String entityName, @Nonnull String aspectName, + @Nonnull final AspectSpec aspectSpec, @Nullable final RecordTemplate oldAspectValue, + @Nullable final RecordTemplate newAspectValue, @Nullable final SystemMetadata oldSystemMetadata, + @Nullable final SystemMetadata newSystemMetadata, @Nonnull AuditStamp auditStamp, @Nonnull final ChangeType changeType); + + RecordTemplate getLatestAspect(@Nonnull final Urn urn, @Nonnull final String aspectName); + + @Deprecated + void ingestEntities(@Nonnull final List entities, @Nonnull final AuditStamp auditStamp, + @Nonnull final List systemMetadata); + + @Deprecated + void ingestEntity(Entity entity, AuditStamp auditStamp); + + @Deprecated + void ingestEntity(@Nonnull Entity entity, @Nonnull AuditStamp auditStamp, + @Nonnull SystemMetadata systemMetadata); + + @Deprecated + Snapshot buildSnapshot(@Nonnull final Urn urn, @Nonnull final RecordTemplate aspectValue); + + void setRetentionService(RetentionService retentionService); + + AspectSpec getKeyAspectSpec(@Nonnull final Urn urn); + + Optional getAspectSpec(@Nonnull final String entityName, @Nonnull final String aspectName); + + String getKeyAspectName(@Nonnull final Urn urn); + + List> generateDefaultAspectsIfMissing(@Nonnull final Urn urn, + Set includedAspects); + + AspectSpec getKeyAspectSpec(@Nonnull final String entityName); + + Set getEntityAspectNames(final String entityName); + + EntityRegistry getEntityRegistry(); + + RollbackResult deleteAspect(String urn, String aspectName, @Nonnull Map conditions, boolean hardDelete); + + RollbackRunResult deleteUrn(Urn urn); + + RollbackRunResult rollbackRun(List aspectRows, String runId, boolean hardDelete); + + RollbackRunResult rollbackWithConditions(List aspectRows, Map conditions, boolean hardDelete); + + IngestProposalResult ingestProposal(@Nonnull MetadataChangeProposal mcp, + AuditStamp auditStamp, final boolean async); + + Boolean exists(Urn urn); + + Boolean isSoftDeleted(@Nonnull final Urn urn); + + void setWritable(boolean canWrite); + + BrowsePaths buildDefaultBrowsePath(final @Nonnull Urn urn) throws URISyntaxException; + + /** + * Builds the default browse path V2 aspects for all entities. + * + * This method currently supports datasets, charts, dashboards, and data jobs best. Everything else + * will have a basic "Default" folder added to their browsePathV2. + */ + @Nonnull + BrowsePathsV2 buildDefaultBrowsePathV2(final @Nonnull Urn urn, boolean useContainerPaths) throws URISyntaxException; +} diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/IngestProposalResult.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/IngestProposalResult.java new file mode 100644 index 0000000000000..27c51e050deff --- /dev/null +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/IngestProposalResult.java @@ -0,0 +1,12 @@ +package com.linkedin.metadata.entity; + +import com.linkedin.common.urn.Urn; +import lombok.Value; + + +@Value +public class IngestProposalResult { + Urn urn; + boolean didUpdate; + boolean queued; +} diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ListResult.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/ListResult.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/entity/ListResult.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/entity/ListResult.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/RetentionService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/RetentionService.java similarity index 99% rename from metadata-io/src/main/java/com/linkedin/metadata/entity/RetentionService.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/entity/RetentionService.java index e4cdb5f531b93..a27cb8076721a 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/RetentionService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/RetentionService.java @@ -30,7 +30,7 @@ /** - * Service coupled with an {@link EntityService} to handle aspect record retention. + * Service coupled with an {@link EntityServiceImpl} to handle aspect record retention. * * TODO: This class is abstract with storage-specific implementations. It'd be nice to pull storage and retention * concerns apart, let (into {@link AspectDao}) deal with storage, and merge all retention concerns into a single diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/RollbackResult.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/RollbackResult.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/entity/RollbackResult.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/entity/RollbackResult.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/RollbackRunResult.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/RollbackRunResult.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/entity/RollbackRunResult.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/entity/RollbackRunResult.java diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/UpdateAspectResult.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/UpdateAspectResult.java new file mode 100644 index 0000000000000..68ecdbd87dd16 --- /dev/null +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/UpdateAspectResult.java @@ -0,0 +1,21 @@ +package com.linkedin.metadata.entity; + +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.mxe.MetadataAuditOperation; +import com.linkedin.mxe.SystemMetadata; +import lombok.Value; + + +@Value +public class UpdateAspectResult { + Urn urn; + RecordTemplate oldValue; + RecordTemplate newValue; + SystemMetadata oldSystemMetadata; + SystemMetadata newSystemMetadata; + MetadataAuditOperation operation; + AuditStamp auditStamp; + long maxVersion; +} diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/restoreindices/RestoreIndicesArgs.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/restoreindices/RestoreIndicesArgs.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/entity/restoreindices/RestoreIndicesArgs.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/entity/restoreindices/RestoreIndicesArgs.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/restoreindices/RestoreIndicesResult.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/restoreindices/RestoreIndicesResult.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/entity/restoreindices/RestoreIndicesResult.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/entity/restoreindices/RestoreIndicesResult.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/retention/BulkApplyRetentionArgs.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/retention/BulkApplyRetentionArgs.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/entity/retention/BulkApplyRetentionArgs.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/entity/retention/BulkApplyRetentionArgs.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/retention/BulkApplyRetentionResult.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/retention/BulkApplyRetentionResult.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/entity/retention/BulkApplyRetentionResult.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/entity/retention/BulkApplyRetentionResult.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/Edge.java b/metadata-service/services/src/main/java/com/linkedin/metadata/graph/Edge.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/graph/Edge.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/graph/Edge.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/GraphClient.java b/metadata-service/services/src/main/java/com/linkedin/metadata/graph/GraphClient.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/graph/GraphClient.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/graph/GraphClient.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/GraphFilters.java b/metadata-service/services/src/main/java/com/linkedin/metadata/graph/GraphFilters.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/graph/GraphFilters.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/graph/GraphFilters.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/GraphIndexUtils.java b/metadata-service/services/src/main/java/com/linkedin/metadata/graph/GraphIndexUtils.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/graph/GraphIndexUtils.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/graph/GraphIndexUtils.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/GraphService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/graph/GraphService.java similarity index 93% rename from metadata-io/src/main/java/com/linkedin/metadata/graph/GraphService.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/graph/GraphService.java index a374fb480154b..6f0ac4bc2f904 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/GraphService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/graph/GraphService.java @@ -17,9 +17,6 @@ import javax.annotation.Nullable; import org.apache.commons.collections.CollectionUtils; -import static com.linkedin.metadata.search.utils.QueryUtils.*; - - public interface GraphService { /** * Return lineage registry to construct graph index @@ -80,7 +77,7 @@ public interface GraphService { * findRelatedEntities(null, EMPTY_FILTER, null, EMPTY_FILTER, ["HasOwner"], RelationshipFilter.setDirection(RelationshipDirection.INCOMING), 0, 100) * - RelatedEntity("HasOwner", "dataset one") * - * Calling this method with {@link com.linkedin.metadata.query.RelationshipDirection} `UNDIRECTED` in `relationshipFilter` + * Calling this method with {@link RelationshipDirection} `UNDIRECTED` in `relationshipFilter` * is equivalent to the union of `OUTGOING` and `INCOMING` (without duplicates). * * Example III: @@ -180,9 +177,9 @@ default EntityLineageResult getLineage(@Nonnull Urn entityUrn, @Nonnull LineageD edgesByDirection.get(true).stream().map(LineageRegistry.EdgeInfo::getType).collect(Collectors.toSet())); // Fetch outgoing edges RelatedEntitiesResult outgoingEdges = - findRelatedEntities(null, newFilter("urn", entityUrn.toString()), graphFilters.getAllowedEntityTypes(), + findRelatedEntities(null, QueryUtils.newFilter("urn", entityUrn.toString()), graphFilters.getAllowedEntityTypes(), QueryUtils.EMPTY_FILTER, - relationshipTypes, newRelationshipFilter(QueryUtils.EMPTY_FILTER, RelationshipDirection.OUTGOING), offset, + relationshipTypes, QueryUtils.newRelationshipFilter(QueryUtils.EMPTY_FILTER, RelationshipDirection.OUTGOING), offset, count); // Update offset and count to fetch the correct number of incoming edges below @@ -206,9 +203,9 @@ relationshipTypes, newRelationshipFilter(QueryUtils.EMPTY_FILTER, RelationshipDi List relationshipTypes = edgesByDirection.get(false).stream().map(LineageRegistry.EdgeInfo::getType).collect(Collectors.toList()); RelatedEntitiesResult incomingEdges = - findRelatedEntities(null, newFilter("urn", entityUrn.toString()), graphFilters.getAllowedEntityTypes(), + findRelatedEntities(null, QueryUtils.newFilter("urn", entityUrn.toString()), graphFilters.getAllowedEntityTypes(), QueryUtils.EMPTY_FILTER, - relationshipTypes, newRelationshipFilter(QueryUtils.EMPTY_FILTER, RelationshipDirection.INCOMING), offset, + relationshipTypes, QueryUtils.newRelationshipFilter(QueryUtils.EMPTY_FILTER, RelationshipDirection.INCOMING), offset, count); result.setTotal(result.getTotal() + incomingEdges.getTotal()); incomingEdges.getEntities().forEach(entity -> { @@ -238,7 +235,7 @@ relationshipTypes, newRelationshipFilter(QueryUtils.EMPTY_FILTER, RelationshipDi * * An empty list of relationship types removes nothing from the node. * - * Calling this method with a {@link com.linkedin.metadata.query.RelationshipDirection} `UNDIRECTED` in `relationshipFilter` + * Calling this method with a {@link RelationshipDirection} `UNDIRECTED` in `relationshipFilter` * is equivalent to the union of `OUTGOING` and `INCOMING` (without duplicates). */ void removeEdgesFromNode(@Nonnull final Urn urn, @Nonnull final List relationshipTypes, diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/RelatedEntitiesResult.java b/metadata-service/services/src/main/java/com/linkedin/metadata/graph/RelatedEntitiesResult.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/graph/RelatedEntitiesResult.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/graph/RelatedEntitiesResult.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/RelatedEntity.java b/metadata-service/services/src/main/java/com/linkedin/metadata/graph/RelatedEntity.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/graph/RelatedEntity.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/graph/RelatedEntity.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/RecommendationsService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/RecommendationsService.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/recommendation/RecommendationsService.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/RecommendationsService.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/DomainsCandidateSource.java b/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/DomainsCandidateSource.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/DomainsCandidateSource.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/DomainsCandidateSource.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/EntitySearchAggregationSource.java b/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/EntitySearchAggregationSource.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/EntitySearchAggregationSource.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/EntitySearchAggregationSource.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlySearchedSource.java b/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlySearchedSource.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlySearchedSource.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecentlySearchedSource.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecommendationSource.java b/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecommendationSource.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecommendationSource.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecommendationSource.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecommendationUtils.java b/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecommendationUtils.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecommendationUtils.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/RecommendationUtils.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/TopPlatformsSource.java b/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/TopPlatformsSource.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/TopPlatformsSource.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/TopPlatformsSource.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/TopTagsSource.java b/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/TopTagsSource.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/TopTagsSource.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/TopTagsSource.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/TopTermsSource.java b/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/TopTermsSource.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/recommendation/candidatesource/TopTermsSource.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/TopTermsSource.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/ranker/RecommendationModuleRanker.java b/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/ranker/RecommendationModuleRanker.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/recommendation/ranker/RecommendationModuleRanker.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/ranker/RecommendationModuleRanker.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/recommendation/ranker/SimpleRecommendationRanker.java b/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/ranker/SimpleRecommendationRanker.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/recommendation/ranker/SimpleRecommendationRanker.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/ranker/SimpleRecommendationRanker.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/schema/registry/SchemaRegistryService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/registry/SchemaRegistryService.java similarity index 89% rename from metadata-io/src/main/java/com/linkedin/metadata/schema/registry/SchemaRegistryService.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/registry/SchemaRegistryService.java index e82b30cc7abb8..0a0be60969486 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/schema/registry/SchemaRegistryService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/registry/SchemaRegistryService.java @@ -1,4 +1,4 @@ -package com.linkedin.metadata.schema.registry; +package com.linkedin.metadata.registry; import java.util.Optional; import org.apache.avro.Schema; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/schema/registry/SchemaRegistryServiceImpl.java b/metadata-service/services/src/main/java/com/linkedin/metadata/registry/SchemaRegistryServiceImpl.java similarity index 98% rename from metadata-io/src/main/java/com/linkedin/metadata/schema/registry/SchemaRegistryServiceImpl.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/registry/SchemaRegistryServiceImpl.java index 1f0f07c40fb07..8f7403c6aa428 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/schema/registry/SchemaRegistryServiceImpl.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/registry/SchemaRegistryServiceImpl.java @@ -1,4 +1,4 @@ -package com.linkedin.metadata.schema.registry; +package com.linkedin.metadata.registry; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/resource/ResourceReference.java b/metadata-service/services/src/main/java/com/linkedin/metadata/resource/ResourceReference.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/resource/ResourceReference.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/resource/ResourceReference.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/resource/SubResourceType.java b/metadata-service/services/src/main/java/com/linkedin/metadata/resource/SubResourceType.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/resource/SubResourceType.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/resource/SubResourceType.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/EntitySearchService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java similarity index 97% rename from metadata-io/src/main/java/com/linkedin/metadata/search/EntitySearchService.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java index 4e1b8b9f1d447..de5bdb62f201b 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/EntitySearchService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java @@ -71,7 +71,7 @@ public interface EntitySearchService { * @param from index to start the search from * @param size the number of search hits to return * @param searchFlags flags controlling search options - * @return a {@link com.linkedin.metadata.dao.SearchResult} that contains a list of matched documents and related search result metadata + * @return a {@link SearchResult} that contains a list of matched documents and related search result metadata */ @Nonnull SearchResult search(@Nonnull String entityName, @Nonnull String input, @Nullable Filter postFilters, @@ -92,7 +92,7 @@ SearchResult search(@Nonnull String entityName, @Nonnull String input, @Nullable * @param size the number of search hits to return * @param searchFlags flags controlling search options * @param facets list of facets we want aggregations for - * @return a {@link com.linkedin.metadata.dao.SearchResult} that contains a list of matched documents and related search result metadata + * @return a {@link SearchResult} that contains a list of matched documents and related search result metadata */ @Nonnull SearchResult search(@Nonnull String entityName, @Nonnull String input, @Nullable Filter postFilters, diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/QueryUtils.java b/metadata-service/services/src/main/java/com/linkedin/metadata/search/utils/QueryUtils.java similarity index 98% rename from metadata-io/src/main/java/com/linkedin/metadata/search/utils/QueryUtils.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/search/utils/QueryUtils.java index 0e5a285c28dbd..31b94425d6815 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/QueryUtils.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/search/utils/QueryUtils.java @@ -5,7 +5,6 @@ import com.linkedin.data.template.RecordTemplate; import com.linkedin.data.template.StringArray; import com.linkedin.metadata.aspect.AspectVersion; -import com.linkedin.metadata.dao.BaseReadDAO; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; @@ -23,6 +22,8 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import static com.linkedin.metadata.Constants.*; + public class QueryUtils { @@ -83,7 +84,7 @@ public static Filter filterOrDefaultEmptyFilter(@Nullable Filter filter) { public static Set latestAspectVersions(@Nonnull Set> aspectClasses) { return aspectClasses.stream() .map(aspectClass -> new AspectVersion().setAspect(ModelUtils.getAspectName(aspectClass)) - .setVersion(BaseReadDAO.LATEST_VERSION)) + .setVersion(LATEST_VERSION)) .collect(Collectors.toSet()); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/secret/SecretService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/secret/SecretService.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/secret/SecretService.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/secret/SecretService.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/BaseService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/BaseService.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/service/BaseService.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/service/BaseService.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/DataProductService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/DataProductService.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/service/DataProductService.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/service/DataProductService.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/DomainService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/DomainService.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/service/DomainService.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/service/DomainService.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/GlossaryTermService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/GlossaryTermService.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/service/GlossaryTermService.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/service/GlossaryTermService.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/LineageService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/LineageService.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/service/LineageService.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/service/LineageService.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/OwnerService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/OwnerService.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/service/OwnerService.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/service/OwnerService.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/OwnershipTypeService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/OwnershipTypeService.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/service/OwnershipTypeService.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/service/OwnershipTypeService.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/QueryService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/QueryService.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/service/QueryService.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/service/QueryService.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/SettingsService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/SettingsService.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/service/SettingsService.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/service/SettingsService.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/TagService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/TagService.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/service/TagService.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/service/TagService.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/ViewService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/ViewService.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/service/ViewService.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/service/ViewService.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/shared/ValidationUtils.java b/metadata-service/services/src/main/java/com/linkedin/metadata/shared/ValidationUtils.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/shared/ValidationUtils.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/shared/ValidationUtils.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/SystemMetadataService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/systemmetadata/SystemMetadataService.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/SystemMetadataService.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/systemmetadata/SystemMetadataService.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/SemanticVersion.java b/metadata-service/services/src/main/java/com/linkedin/metadata/timeline/SemanticVersion.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/timeline/SemanticVersion.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/timeline/SemanticVersion.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/TimelineService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/timeline/TimelineService.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/timeline/TimelineService.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/timeline/TimelineService.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/data/ChangeCategory.java b/metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/ChangeCategory.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/timeline/data/ChangeCategory.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/ChangeCategory.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/data/ChangeEvent.java b/metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/ChangeEvent.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/timeline/data/ChangeEvent.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/ChangeEvent.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/data/ChangeOperation.java b/metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/ChangeOperation.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/timeline/data/ChangeOperation.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/ChangeOperation.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/data/ChangeTransaction.java b/metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/ChangeTransaction.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/timeline/data/ChangeTransaction.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/ChangeTransaction.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/data/PatchOperation.java b/metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/PatchOperation.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/timeline/data/PatchOperation.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/PatchOperation.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/data/SemanticChangeType.java b/metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/SemanticChangeType.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/timeline/data/SemanticChangeType.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/SemanticChangeType.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/data/SemanticDifference.java b/metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/SemanticDifference.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/timeline/data/SemanticDifference.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/SemanticDifference.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java similarity index 100% rename from metadata-io/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java rename to metadata-service/services/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java diff --git a/metadata-io/src/test/java/com/linkedin/metadata/service/DomainServiceTest.java b/metadata-service/services/src/test/java/com/linkedin/metadata/service/DomainServiceTest.java similarity index 98% rename from metadata-io/src/test/java/com/linkedin/metadata/service/DomainServiceTest.java rename to metadata-service/services/src/test/java/com/linkedin/metadata/service/DomainServiceTest.java index a600ec610ba99..4338d883ece1d 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/service/DomainServiceTest.java +++ b/metadata-service/services/src/test/java/com/linkedin/metadata/service/DomainServiceTest.java @@ -2,6 +2,9 @@ import com.datahub.authentication.Actor; import com.datahub.authentication.ActorType; +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; @@ -12,6 +15,7 @@ import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; import com.linkedin.metadata.resource.ResourceReference; import com.linkedin.metadata.utils.GenericRecordUtils; @@ -20,12 +24,8 @@ import java.util.List; import javax.annotation.Nullable; import org.mockito.Mockito; -import org.testcontainers.shaded.com.google.common.collect.ImmutableList; -import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; import org.testng.Assert; import org.testng.annotations.Test; -import com.linkedin.entity.client.EntityClient; -import com.datahub.authentication.Authentication; public class DomainServiceTest { diff --git a/metadata-io/src/test/java/com/linkedin/metadata/service/GlossaryTermServiceTest.java b/metadata-service/services/src/test/java/com/linkedin/metadata/service/GlossaryTermServiceTest.java similarity index 99% rename from metadata-io/src/test/java/com/linkedin/metadata/service/GlossaryTermServiceTest.java rename to metadata-service/services/src/test/java/com/linkedin/metadata/service/GlossaryTermServiceTest.java index f2e25792fe8bc..567a457efcf93 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/service/GlossaryTermServiceTest.java +++ b/metadata-service/services/src/test/java/com/linkedin/metadata/service/GlossaryTermServiceTest.java @@ -3,6 +3,8 @@ import com.datahub.authentication.Actor; import com.datahub.authentication.ActorType; import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.linkedin.common.GlossaryTermAssociation; import com.linkedin.common.GlossaryTermAssociationArray; @@ -28,8 +30,6 @@ import java.util.List; import javax.annotation.Nullable; import org.mockito.Mockito; -import org.testcontainers.shaded.com.google.common.collect.ImmutableList; -import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/service/LineageServiceTest.java b/metadata-service/services/src/test/java/com/linkedin/metadata/service/LineageServiceTest.java similarity index 99% rename from metadata-io/src/test/java/com/linkedin/metadata/service/LineageServiceTest.java rename to metadata-service/services/src/test/java/com/linkedin/metadata/service/LineageServiceTest.java index db5e0a88bf25e..9df8b9ecf46e8 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/service/LineageServiceTest.java +++ b/metadata-service/services/src/test/java/com/linkedin/metadata/service/LineageServiceTest.java @@ -3,6 +3,7 @@ import com.datahub.authentication.Actor; import com.datahub.authentication.ActorType; import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.linkedin.chart.ChartDataSourceType; import com.linkedin.chart.ChartDataSourceTypeArray; @@ -35,19 +36,17 @@ import com.linkedin.metadata.Constants; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; -import org.joda.time.DateTimeUtils; -import org.mockito.Mockito; -import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import javax.annotation.Nonnull; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import javax.annotation.Nonnull; +import org.joda.time.DateTimeUtils; +import org.mockito.Mockito; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; -import static org.testng.Assert.assertThrows; +import static org.testng.Assert.*; public class LineageServiceTest { private static AuditStamp _auditStamp; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/service/OwnerServiceTest.java b/metadata-service/services/src/test/java/com/linkedin/metadata/service/OwnerServiceTest.java similarity index 98% rename from metadata-io/src/test/java/com/linkedin/metadata/service/OwnerServiceTest.java rename to metadata-service/services/src/test/java/com/linkedin/metadata/service/OwnerServiceTest.java index 9247837fe0b32..c23a151e52734 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/service/OwnerServiceTest.java +++ b/metadata-service/services/src/test/java/com/linkedin/metadata/service/OwnerServiceTest.java @@ -3,6 +3,8 @@ import com.datahub.authentication.Actor; import com.datahub.authentication.ActorType; import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.linkedin.common.Owner; import com.linkedin.common.OwnerArray; @@ -24,8 +26,6 @@ import java.util.List; import javax.annotation.Nullable; import org.mockito.Mockito; -import org.testcontainers.shaded.com.google.common.collect.ImmutableList; -import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/service/OwnershipTypeServiceTest.java b/metadata-service/services/src/test/java/com/linkedin/metadata/service/OwnershipTypeServiceTest.java similarity index 99% rename from metadata-io/src/test/java/com/linkedin/metadata/service/OwnershipTypeServiceTest.java rename to metadata-service/services/src/test/java/com/linkedin/metadata/service/OwnershipTypeServiceTest.java index fd79ab7b99ccd..dcb4a745732b2 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/service/OwnershipTypeServiceTest.java +++ b/metadata-service/services/src/test/java/com/linkedin/metadata/service/OwnershipTypeServiceTest.java @@ -3,6 +3,7 @@ import com.datahub.authentication.Actor; import com.datahub.authentication.ActorType; import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; @@ -19,7 +20,6 @@ import com.linkedin.ownership.OwnershipTypeInfo; import com.linkedin.r2.RemoteInvocationException; import org.mockito.Mockito; -import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/service/QueryServiceTest.java b/metadata-service/services/src/test/java/com/linkedin/metadata/service/QueryServiceTest.java similarity index 100% rename from metadata-io/src/test/java/com/linkedin/metadata/service/QueryServiceTest.java rename to metadata-service/services/src/test/java/com/linkedin/metadata/service/QueryServiceTest.java diff --git a/metadata-io/src/test/java/com/linkedin/metadata/service/SettingsServiceTest.java b/metadata-service/services/src/test/java/com/linkedin/metadata/service/SettingsServiceTest.java similarity index 100% rename from metadata-io/src/test/java/com/linkedin/metadata/service/SettingsServiceTest.java rename to metadata-service/services/src/test/java/com/linkedin/metadata/service/SettingsServiceTest.java diff --git a/metadata-io/src/test/java/com/linkedin/metadata/service/TagServiceTest.java b/metadata-service/services/src/test/java/com/linkedin/metadata/service/TagServiceTest.java similarity index 99% rename from metadata-io/src/test/java/com/linkedin/metadata/service/TagServiceTest.java rename to metadata-service/services/src/test/java/com/linkedin/metadata/service/TagServiceTest.java index cc1e4831b30b0..125265540dc77 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/service/TagServiceTest.java +++ b/metadata-service/services/src/test/java/com/linkedin/metadata/service/TagServiceTest.java @@ -3,6 +3,8 @@ import com.datahub.authentication.Actor; import com.datahub.authentication.ActorType; import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.linkedin.common.GlobalTags; import com.linkedin.common.TagAssociation; @@ -28,8 +30,6 @@ import java.util.List; import javax.annotation.Nullable; import org.mockito.Mockito; -import org.testcontainers.shaded.com.google.common.collect.ImmutableList; -import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/service/ViewServiceTest.java b/metadata-service/services/src/test/java/com/linkedin/metadata/service/ViewServiceTest.java similarity index 99% rename from metadata-io/src/test/java/com/linkedin/metadata/service/ViewServiceTest.java rename to metadata-service/services/src/test/java/com/linkedin/metadata/service/ViewServiceTest.java index 79449c1ff62c1..5841717e7db93 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/service/ViewServiceTest.java +++ b/metadata-service/services/src/test/java/com/linkedin/metadata/service/ViewServiceTest.java @@ -3,6 +3,8 @@ import com.datahub.authentication.Actor; import com.datahub.authentication.ActorType; import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; @@ -28,8 +30,6 @@ import com.linkedin.view.DataHubViewType; import java.util.Collections; import org.mockito.Mockito; -import org.testcontainers.shaded.com.google.common.collect.ImmutableList; -import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/metadata-service/servlet/src/main/java/com/datahub/gms/servlet/ConfigSearchExport.java b/metadata-service/servlet/src/main/java/com/datahub/gms/servlet/ConfigSearchExport.java index c14d7cbf1ab2f..2c26c00e9c4d6 100644 --- a/metadata-service/servlet/src/main/java/com/datahub/gms/servlet/ConfigSearchExport.java +++ b/metadata-service/servlet/src/main/java/com/datahub/gms/servlet/ConfigSearchExport.java @@ -1,9 +1,9 @@ package com.datahub.gms.servlet; +import com.linkedin.metadata.config.search.SearchConfiguration; import com.datahub.gms.util.CSVWriter; import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; import com.linkedin.gms.factory.config.ConfigurationProvider; -import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.query.SearchFlags; diff --git a/settings.gradle b/settings.gradle index e4d1702829bc8..f326cbf10b96b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -60,4 +60,5 @@ include 'metadata-service:schema-registry-api' include 'metadata-service:schema-registry-servlet' include 'metadata-integration:java:examples' include 'mock-entity-registry' - +include 'metadata-service:services' +include 'metadata-service:configuration' From d734b2849e514b1c539d195123602b11cefdfc20 Mon Sep 17 00:00:00 2001 From: Ellie O'Neil <110510035+eboneil@users.noreply.github.com> Date: Wed, 19 Jul 2023 22:36:26 -0400 Subject: [PATCH 162/222] feat(ingest/mysql): Add estimate row count for mysql (#8420) --- .../ingestion/source/ge_data_profiler.py | 29 +- .../ingestion/source/ge_profiling_config.py | 2 +- ..._profile_table_row_count_estimate_only.yml | 14 + .../mysql_table_row_count_estimate_only.json | 404 ++++++++++++++++++ .../tests/integration/mysql/test_mysql.py | 4 + 5 files changed, 443 insertions(+), 10 deletions(-) create mode 100644 metadata-ingestion/tests/integration/mysql/mysql_profile_table_row_count_estimate_only.yml create mode 100644 metadata-ingestion/tests/integration/mysql/mysql_table_row_count_estimate_only.json diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py index ab259b9b16dd2..4ea721f6fd0cc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py @@ -359,21 +359,32 @@ def _get_column_cardinality( @_run_with_query_combiner def _get_dataset_rows(self, dataset_profile: DatasetProfileClass) -> None: - if ( - self.config.profile_table_row_count_estimate_only - and self.dataset.engine.dialect.name.lower() == "postgresql" - ): + if self.config.profile_table_row_count_estimate_only: schema_name = self.dataset_name.split(".")[1] table_name = self.dataset_name.split(".")[2] logger.debug( f"Getting estimated rowcounts for table:{self.dataset_name}, schema:{schema_name}, table:{table_name}" ) + + dialect_name = self.dataset.engine.dialect.name.lower() + if dialect_name == "postgresql": + get_estimate_script = sa.text( + f"SELECT c.reltuples AS estimate FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relname = '{table_name}' AND n.nspname = '{schema_name}'" + ) + elif dialect_name == "mysql": + get_estimate_script = sa.text( + f"SELECT table_rows AS estimate FROM information_schema.tables WHERE table_schema = '{schema_name}' AND table_name = '{table_name}'" + ) + else: + logger.debug( + f"Dialect {dialect_name} not supported for feature " + f"profile_table_row_count_estimate_only. Proceeding with full row count." + ) + dataset_profile.rowCount = self.dataset.get_row_count() + return + dataset_profile.rowCount = int( - self.dataset.engine.execute( - sa.text( - f"SELECT c.reltuples AS estimate FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relname = '{table_name}' AND n.nspname = '{schema_name}'" - ) - ).scalar() + self.dataset.engine.execute(get_estimate_script).scalar() ) else: dataset_profile.rowCount = self.dataset.get_row_count() diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py b/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py index 3f61a99bfd585..8c5f1646c1d67 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py @@ -118,7 +118,7 @@ class GEProfilingConfig(ConfigModel): profile_table_row_count_estimate_only: bool = Field( default=False, description="Use an approximate query for row count. This will be much faster but slightly " - "less accurate. Only supported for Postgres. ", + "less accurate. Only supported for Postgres and MySQL. ", ) # The default of (5 * cpu_count) is adopted from the default max_workers diff --git a/metadata-ingestion/tests/integration/mysql/mysql_profile_table_row_count_estimate_only.yml b/metadata-ingestion/tests/integration/mysql/mysql_profile_table_row_count_estimate_only.yml new file mode 100644 index 0000000000000..d6a9ed85a8fce --- /dev/null +++ b/metadata-ingestion/tests/integration/mysql/mysql_profile_table_row_count_estimate_only.yml @@ -0,0 +1,14 @@ +source: + type: mysql + config: + username: root + password: example + host_port: localhost:53307 + database: northwind + profiling: + enabled: True + profile_table_row_count_estimate_only: true +sink: + type: file + config: + filename: "./mysql_mces.json" diff --git a/metadata-ingestion/tests/integration/mysql/mysql_table_row_count_estimate_only.json b/metadata-ingestion/tests/integration/mysql/mysql_table_row_count_estimate_only.json new file mode 100644 index 0000000000000..e668525b930af --- /dev/null +++ b/metadata-ingestion/tests/integration/mysql/mysql_table_row_count_estimate_only.json @@ -0,0 +1,404 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "mysql", + "env": "PROD", + "database": "northwind" + }, + "name": "northwind" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mysql" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Database" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "customers", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "northwind.customers", + "platform": "urn:li:dataPlatform:mysql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "company", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=50)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=50)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=50)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email_address", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=50)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", + "urn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "orders", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "northwind.orders", + "platform": "urn:li:dataPlatform:mysql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "description", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=50)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + } + ], + "foreignKeys": [ + { + "name": "fk_order_customer", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),id)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),customer_id)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", + "urn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/mysql/test_mysql.py b/metadata-ingestion/tests/integration/mysql/test_mysql.py index bf5806bc57b13..8c8626a2d2297 100644 --- a/metadata-ingestion/tests/integration/mysql/test_mysql.py +++ b/metadata-ingestion/tests/integration/mysql/test_mysql.py @@ -48,6 +48,10 @@ def mysql_runner(docker_compose_runner, pytestconfig, test_resources_dir): ("mysql_to_file_with_db.yml", "mysql_mces_with_db_golden.json"), ("mysql_to_file_no_db.yml", "mysql_mces_no_db_golden.json"), ("mysql_profile_table_level_only.yml", "mysql_table_level_only.json"), + ( + "mysql_profile_table_row_count_estimate_only.yml", + "mysql_table_row_count_estimate_only.json", + ), ], ) @freeze_time(FROZEN_TIME) From 9df70d7355e2c66a29665d8cae082f237f5d8470 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Thu, 20 Jul 2023 08:25:30 +0530 Subject: [PATCH 163/222] ingest(elasticsearch): add basic profiling (#8351) --- .../docs/transformer/dataset_transformer.md | 28 +++++ metadata-ingestion/setup.py | 1 + .../ingestion/source/elastic_search.py | 101 +++++++++++++++++- .../ingestion/transformer/add_dataset_tags.py | 42 ++------ .../transformer/dataset_transformer.py | 58 +++++++++- .../transformer/extract_dataset_tags.py | 62 +++++++++++ .../tests/unit/test_elasticsearch_source.py | 28 +++++ .../tests/unit/test_transform_dataset.py | 51 +++++++++ 8 files changed, 332 insertions(+), 39 deletions(-) create mode 100644 metadata-ingestion/src/datahub/ingestion/transformer/extract_dataset_tags.py diff --git a/metadata-ingestion/docs/transformer/dataset_transformer.md b/metadata-ingestion/docs/transformer/dataset_transformer.md index 753e9c87300d9..cb06656940918 100644 --- a/metadata-ingestion/docs/transformer/dataset_transformer.md +++ b/metadata-ingestion/docs/transformer/dataset_transformer.md @@ -175,6 +175,34 @@ transformers: The main use case of `simple_remove_dataset_ownership` is to remove incorrect owners present in the source. You can use it along with the [Simple Add Dataset ownership](#simple-add-dataset-ownership) to remove wrong owners and add the correct ones. Note that whatever owners you send via `simple_remove_dataset_ownership` will overwrite the owners present in the UI. +## Extract Dataset globalTags +### Config Details +| Field | Required | Type | Default | Description | +|-----------------------------|----------|--------------|---------------|------------------------------------------------------------------| +| `extract_tags_from` | ✅ | string | `urn` | Which field to extract tag from. Currently only `urn` is supported. | +| `extract_tags_regex` | ✅ | string | `.*` | Regex to use to extract tag.| +| `replace_existing` | | boolean | `false` | Whether to remove owners from entity sent by ingestion source. | +| `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | + +Let’s suppose we’d like to add a dataset tags based on part of urn. To do so, we can use the `extract_dataset_tags` transformer that’s included in the ingestion framework. + +The config, which we’d append to our ingestion recipe YAML, would look like this: + + ```yaml + transformers: + - type: "extract_dataset_tags" + config: + extract_tags_from: "urn" + extract_tags_regex: ".([^._]*)_" + ``` + +So if we have input URNs like +- `urn:li:dataset:(urn:li:dataPlatform:kafka,clusterid.USA-ops-team_table1,PROD)` +- `urn:li:dataset:(urn:li:dataPlatform:kafka,clusterid.Canada-marketing_table1,PROD)` + +a tag called `USA-ops-team` and `Canada-marketing` will be added to them respectively. This is helpful in case you are using prefixes in your datasets to segregate different things. Now you can turn that segregation into a tag on your dataset in DataHub for further use. + + ## Simple Add Dataset globalTags ### Config Details | Field | Required | Type | Default | Description | diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 029527ea959d5..36f4151df0359 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -613,6 +613,7 @@ def get_long_description(): "add_dataset_tags = datahub.ingestion.transformer.add_dataset_tags:AddDatasetTags", "simple_add_dataset_tags = datahub.ingestion.transformer.add_dataset_tags:SimpleAddDatasetTags", "pattern_add_dataset_tags = datahub.ingestion.transformer.add_dataset_tags:PatternAddDatasetTags", + "extract_dataset_tags = datahub.ingestion.transformer.extract_dataset_tags:ExtractDatasetTags", "add_dataset_terms = datahub.ingestion.transformer.add_dataset_terms:AddDatasetTerms", "simple_add_dataset_terms = datahub.ingestion.transformer.add_dataset_terms:SimpleAddDatasetTerms", "pattern_add_dataset_terms = datahub.ingestion.transformer.add_dataset_terms:PatternAddDatasetTerms", diff --git a/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py b/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py index ef0f4ed21f83c..ab6aa18dac4d2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py +++ b/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py @@ -1,5 +1,7 @@ import json import logging +import re +import time from collections import defaultdict from dataclasses import dataclass, field from hashlib import md5 @@ -9,7 +11,7 @@ from pydantic import validator from pydantic.fields import Field -from datahub.configuration.common import AllowDenyPattern +from datahub.configuration.common import AllowDenyPattern, ConfigModel from datahub.configuration.source_common import ( EnvConfigMixin, PlatformInstanceConfigMixin, @@ -45,6 +47,7 @@ BooleanTypeClass, BytesTypeClass, DataPlatformInstanceClass, + DatasetProfileClass, DatasetPropertiesClass, DateTypeClass, NullTypeClass, @@ -56,6 +59,7 @@ SubTypesClass, ) from datahub.utilities.config_clean import remove_protocol +from datahub.utilities.urns.dataset_urn import DatasetUrn logger = logging.getLogger(__name__) @@ -191,6 +195,43 @@ def report_dropped(self, index: str) -> None: self.filtered.append(index) +class ElasticProfiling(ConfigModel): + enabled: bool = Field( + default=False, + description="Whether to enable profiling for the elastic search source.", + ) + + +class CollapseUrns(ConfigModel): + urns_suffix_regex: List[str] = Field( + default_factory=list, + description="""List of regex patterns to remove from the name of the URN. All of the indices before removal of URNs are considered as the same dataset. These are applied in order for each URN. + The main case where you would want to have multiple of these if the name where you are trying to remove suffix from have different formats. + e.g. ending with -YYYY-MM-DD as well as ending -epochtime would require you to have 2 regex patterns to remove the suffixes across all URNs.""", + ) + + +def collapse_name(name: str, collapse_urns: CollapseUrns) -> str: + for suffix in collapse_urns.urns_suffix_regex: + name = re.sub(suffix, "", name) + return name + + +def collapse_urn(urn: str, collapse_urns: CollapseUrns) -> str: + if len(collapse_urns.urns_suffix_regex) == 0: + return urn + urn_obj = DatasetUrn.create_from_string(urn) + name = collapse_name(name=urn_obj.get_dataset_name(), collapse_urns=collapse_urns) + data_platform_urn = urn_obj.get_data_platform_urn() + return str( + DatasetUrn.create_from_ids( + platform_id=data_platform_urn.get_entity_id_as_string(), + table_name=name, + env=urn_obj.get_env(), + ) + ) + + class ElasticsearchSourceConfig(PlatformInstanceConfigMixin, EnvConfigMixin): host: str = Field( default="localhost:9200", description="The elastic search host URI." @@ -249,6 +290,13 @@ class ElasticsearchSourceConfig(PlatformInstanceConfigMixin, EnvConfigMixin): description="The regex patterns for filtering index templates to ingest.", ) + profiling: ElasticProfiling = Field( + default_factory=ElasticProfiling, + ) + collapse_urns: CollapseUrns = Field( + default_factory=CollapseUrns, + ) + @validator("host") def host_colon_port_comma(cls, host_val: str) -> str: for entry in host_val.split(","): @@ -295,6 +343,7 @@ def __init__(self, config: ElasticsearchSourceConfig, ctx: PipelineContext): self.report = ElasticsearchSourceReport() self.data_stream_partition_count: Dict[str, int] = defaultdict(int) self.platform: str = "elasticsearch" + self.profiling_info: Dict[str, DatasetProfileClass] = {} @classmethod def create( @@ -317,6 +366,12 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: yield mcp.as_workunit() else: self.report.report_dropped(index) + for urn, profiling_info in self.profiling_info.items(): + yield MetadataChangeProposalWrapper( + entityUrn=urn, + aspect=profiling_info, + ).as_workunit() + self.profiling_info = {} for mcp in self._get_data_stream_index_count_mcps(): yield mcp.as_workunit() @@ -337,6 +392,9 @@ def _get_data_stream_index_count_mcps( env=self.source_config.env, platform_instance=self.source_config.platform_instance, ) + dataset_urn = collapse_urn( + urn=dataset_urn, collapse_urns=self.source_config.collapse_urns + ) yield MetadataChangeProposalWrapper( entityUrn=dataset_urn, aspect=DatasetPropertiesClass( @@ -364,6 +422,9 @@ def _extract_mcps( else: raw_index = self.client.indices.get_template(name=index) raw_index_metadata = raw_index[index] + collapsed_index_name = collapse_name( + name=index, collapse_urns=self.source_config.collapse_urns + ) # 1. Construct and emit the schemaMetadata aspect # 1.1 Generate the schema fields from ES mappings. @@ -378,7 +439,7 @@ def _extract_mcps( # 1.2 Generate the SchemaMetadata aspect schema_metadata = SchemaMetadata( - schemaName=index, + schemaName=collapsed_index_name, platform=make_data_platform_urn(self.platform), version=0, hash=md5_hash, @@ -393,6 +454,9 @@ def _extract_mcps( platform_instance=self.source_config.platform_instance, env=self.source_config.env, ) + dataset_urn = collapse_urn( + urn=dataset_urn, collapse_urns=self.source_config.collapse_urns + ) yield MetadataChangeProposalWrapper( entityUrn=dataset_urn, aspect=schema_metadata, @@ -458,6 +522,39 @@ def _extract_mcps( ), ) + if self.source_config.profiling.enabled: + cat_response = self.client.cat.indices( + index=index, params={"format": "json", "bytes": "b"} + ) + if len(cat_response) == 1: + index_res = cat_response[0] + docs_count = int(index_res["docs.count"]) + size = int(index_res["store.size"]) + if len(self.source_config.collapse_urns.urns_suffix_regex) > 0: + if dataset_urn not in self.profiling_info: + self.profiling_info[dataset_urn] = DatasetProfileClass( + timestampMillis=int(time.time() * 1000), + rowCount=docs_count, + columnCount=len(schema_fields), + sizeInBytes=size, + ) + else: + existing_profile = self.profiling_info[dataset_urn] + if existing_profile.rowCount is not None: + docs_count = docs_count + existing_profile.rowCount + if existing_profile.sizeInBytes is not None: + size = size + existing_profile.sizeInBytes + self.profiling_info[dataset_urn] = DatasetProfileClass( + timestampMillis=int(time.time() * 1000), + rowCount=docs_count, + columnCount=len(schema_fields), + sizeInBytes=size, + ) + else: + logger.warning( + "Unexpected response from cat response with multiple rows" + ) + def get_report(self): return self.report diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_tags.py b/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_tags.py index 02df9e77a6a6d..5a276ad899c48 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_tags.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_tags.py @@ -2,13 +2,11 @@ from datahub.configuration.common import ( KeyValuePattern, - TransformerSemantics, TransformerSemanticsConfigModel, ) from datahub.configuration.import_resolver import pydantic_resolve_key from datahub.emitter.mce_builder import Aspect from datahub.ingestion.api.common import PipelineContext -from datahub.ingestion.graph.client import DataHubGraph from datahub.ingestion.transformer.dataset_transformer import DatasetTagsTransformer from datahub.metadata.schema_classes import GlobalTagsClass, TagAssociationClass @@ -35,50 +33,22 @@ def create(cls, config_dict: dict, ctx: PipelineContext) -> "AddDatasetTags": config = AddDatasetTagsConfig.parse_obj(config_dict) return cls(config, ctx) - @staticmethod - def _merge_with_server_global_tags( - graph: DataHubGraph, urn: str, global_tags_aspect: Optional[GlobalTagsClass] - ) -> Optional[GlobalTagsClass]: - if not global_tags_aspect or not global_tags_aspect.tags: - # nothing to add, no need to consult server - return None - - # Merge the transformed tags with existing server tags. - # The transformed tags takes precedence, which may change the tag context. - server_global_tags_aspect = graph.get_tags(entity_urn=urn) - if server_global_tags_aspect: - global_tags_aspect.tags = list( - { - **{tag.tag: tag for tag in server_global_tags_aspect.tags}, - **{tag.tag: tag for tag in global_tags_aspect.tags}, - }.values() - ) - - return global_tags_aspect - def transform_aspect( self, entity_urn: str, aspect_name: str, aspect: Optional[Aspect] ) -> Optional[Aspect]: in_global_tags_aspect: GlobalTagsClass = cast(GlobalTagsClass, aspect) out_global_tags_aspect: GlobalTagsClass = GlobalTagsClass(tags=[]) - # Check if user want to keep existing tags - if in_global_tags_aspect is not None and self.config.replace_existing is False: - out_global_tags_aspect.tags.extend(in_global_tags_aspect.tags) + self.update_if_keep_existing( + self.config, in_global_tags_aspect, out_global_tags_aspect + ) tags_to_add = self.config.get_tags_to_add(entity_urn) if tags_to_add is not None: out_global_tags_aspect.tags.extend(tags_to_add) - if self.config.semantics == TransformerSemantics.PATCH: - assert self.ctx.graph - return cast( - Optional[Aspect], - AddDatasetTags._merge_with_server_global_tags( - self.ctx.graph, entity_urn, out_global_tags_aspect - ), - ) - - return cast(Aspect, out_global_tags_aspect) + return self.get_result_semantics( + self.config, self.ctx.graph, entity_urn, out_global_tags_aspect + ) class SimpleDatasetTagConfig(TransformerSemanticsConfigModel): diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/dataset_transformer.py b/metadata-ingestion/src/datahub/ingestion/transformer/dataset_transformer.py index 22f8a11037137..0753d6c3fd830 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/dataset_transformer.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/dataset_transformer.py @@ -1,11 +1,18 @@ import logging from abc import ABCMeta -from typing import List +from typing import List, Optional, cast +from datahub.configuration.common import ( + TransformerSemantics, + TransformerSemanticsConfigModel, +) +from datahub.emitter.mce_builder import Aspect +from datahub.ingestion.graph.client import DataHubGraph from datahub.ingestion.transformer.base_transformer import ( BaseTransformer, SingleAspectTransformer, ) +from datahub.metadata.schema_classes import GlobalTagsClass log = logging.getLogger(__name__) @@ -39,6 +46,55 @@ class DatasetTagsTransformer(DatasetTransformer, metaclass=ABCMeta): def aspect_name(self) -> str: return "globalTags" + @staticmethod + def merge_with_server_global_tags( + graph: DataHubGraph, urn: str, global_tags_aspect: Optional[GlobalTagsClass] + ) -> Optional[GlobalTagsClass]: + if not global_tags_aspect or not global_tags_aspect.tags: + # nothing to add, no need to consult server + return None + + # Merge the transformed tags with existing server tags. + # The transformed tags takes precedence, which may change the tag context. + server_global_tags_aspect = graph.get_tags(entity_urn=urn) + if server_global_tags_aspect: + global_tags_aspect.tags = list( + { + **{tag.tag: tag for tag in server_global_tags_aspect.tags}, + **{tag.tag: tag for tag in global_tags_aspect.tags}, + }.values() + ) + + return global_tags_aspect + + @staticmethod + def update_if_keep_existing( + config: TransformerSemanticsConfigModel, + in_global_tags_aspect: GlobalTagsClass, + out_global_tags_aspect: GlobalTagsClass, + ) -> None: + """Check if user want to keep existing tags""" + if in_global_tags_aspect is not None and config.replace_existing is False: + out_global_tags_aspect.tags.extend(in_global_tags_aspect.tags) + + @staticmethod + def get_result_semantics( + config: TransformerSemanticsConfigModel, + graph: Optional[DataHubGraph], + urn: str, + out_global_tags_aspect: Optional[GlobalTagsClass], + ) -> Optional[Aspect]: + if config.semantics == TransformerSemantics.PATCH: + assert graph + return cast( + Optional[Aspect], + DatasetTagsTransformer.merge_with_server_global_tags( + graph, urn, out_global_tags_aspect + ), + ) + + return cast(Aspect, out_global_tags_aspect) + class DatasetTermsTransformer(DatasetTransformer, metaclass=ABCMeta): def aspect_name(self) -> str: diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/extract_dataset_tags.py b/metadata-ingestion/src/datahub/ingestion/transformer/extract_dataset_tags.py new file mode 100644 index 0000000000000..25b18f0806fd6 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/transformer/extract_dataset_tags.py @@ -0,0 +1,62 @@ +import re +from enum import Enum +from typing import List, Optional, cast + +from datahub.configuration.common import TransformerSemanticsConfigModel +from datahub.emitter.mce_builder import Aspect +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.transformer.dataset_transformer import DatasetTagsTransformer +from datahub.metadata.schema_classes import GlobalTagsClass, TagAssociationClass +from datahub.utilities.urns.dataset_urn import DatasetUrn + + +class ExtractTagsOption(Enum): + URN = "urn" + + +class ExtractDatasetTagsConfig(TransformerSemanticsConfigModel): + extract_tags_from: ExtractTagsOption = ExtractTagsOption.URN + extract_tags_regex: str + + +class ExtractDatasetTags(DatasetTagsTransformer): + """Transformer that add tags to datasets according to configuration by extracting from metadata. Currently only extracts from name.""" + + def __init__(self, config: ExtractDatasetTagsConfig, ctx: PipelineContext): + super().__init__() + self.ctx: PipelineContext = ctx + self.config: ExtractDatasetTagsConfig = config + + @classmethod + def create(cls, config_dict: dict, ctx: PipelineContext) -> "ExtractDatasetTags": + config = ExtractDatasetTagsConfig.parse_obj(config_dict) + return cls(config, ctx) + + def _get_tags_to_add(self, entity_urn: str) -> List[TagAssociationClass]: + if self.config.extract_tags_from == ExtractTagsOption.URN: + urn = DatasetUrn.create_from_string(entity_urn) + match = re.search(self.config.extract_tags_regex, urn.get_dataset_name()) + if match: + captured_group = match.group(1) + tag = f"urn:li:tag:{captured_group}" + return [TagAssociationClass(tag=tag)] + return [] + else: + raise NotImplementedError() + + def transform_aspect( + self, entity_urn: str, aspect_name: str, aspect: Optional[Aspect] + ) -> Optional[Aspect]: + in_global_tags_aspect: GlobalTagsClass = cast(GlobalTagsClass, aspect) + out_global_tags_aspect: GlobalTagsClass = GlobalTagsClass(tags=[]) + self.update_if_keep_existing( + self.config, in_global_tags_aspect, out_global_tags_aspect + ) + + tags_to_add = self._get_tags_to_add(entity_urn) + if tags_to_add is not None: + out_global_tags_aspect.tags.extend(tags_to_add) + + return self.get_result_semantics( + self.config, self.ctx.graph, entity_urn, out_global_tags_aspect + ) diff --git a/metadata-ingestion/tests/unit/test_elasticsearch_source.py b/metadata-ingestion/tests/unit/test_elasticsearch_source.py index 7a5fc1b58b6b1..cd158b855cdfc 100644 --- a/metadata-ingestion/tests/unit/test_elasticsearch_source.py +++ b/metadata-ingestion/tests/unit/test_elasticsearch_source.py @@ -7,8 +7,10 @@ import pytest from datahub.ingestion.source.elastic_search import ( + CollapseUrns, ElasticsearchSourceConfig, ElasticToSchemaFieldConverter, + collapse_urn, ) from datahub.metadata.com.linkedin.pegasus2avro.schema import SchemaField @@ -2470,3 +2472,29 @@ def test_host_port_parsing() -> None: with pytest.raises(pydantic.ValidationError): ElasticsearchSourceConfig.parse_obj(config_dict) + + +def test_collapse_urns() -> None: + assert ( + collapse_urn( + urn="urn:li:dataset:(urn:li:dataPlatform:elasticsearch,platform1.prefix_datahub_usage_event-000059,PROD)", + collapse_urns=CollapseUrns( + urns_suffix_regex=[ + "-\\d+$", + ] + ), + ) + == "urn:li:dataset:(urn:li:dataPlatform:elasticsearch,platform1.prefix_datahub_usage_event,PROD)" + ) + + assert ( + collapse_urn( + urn="urn:li:dataset:(urn:li:dataPlatform:elasticsearch,platform1.prefix_datahub_usage_event-2023.01.11,PROD)", + collapse_urns=CollapseUrns( + urns_suffix_regex=[ + "-\\d{4}\\.\\d{2}\\.\\d{2}", + ] + ), + ) + == "urn:li:dataset:(urn:li:dataPlatform:elasticsearch,platform1.prefix_datahub_usage_event,PROD)" + ) diff --git a/metadata-ingestion/tests/unit/test_transform_dataset.py b/metadata-ingestion/tests/unit/test_transform_dataset.py index 893431dda6b34..8b2535eea1fe9 100644 --- a/metadata-ingestion/tests/unit/test_transform_dataset.py +++ b/metadata-ingestion/tests/unit/test_transform_dataset.py @@ -61,6 +61,7 @@ SimpleAddDatasetDomain, ) from datahub.ingestion.transformer.dataset_transformer import DatasetTransformer +from datahub.ingestion.transformer.extract_dataset_tags import ExtractDatasetTags from datahub.ingestion.transformer.mark_dataset_status import MarkDatasetStatus from datahub.ingestion.transformer.remove_dataset_ownership import ( SimpleRemoveDatasetOwnership, @@ -288,6 +289,56 @@ def test_simple_dataset_ownership_with_type_transformation(mock_time): assert ownership_aspect.owners[0].type == models.OwnershipTypeClass.PRODUCER +def _test_extract_tags(in_urn: str, regex_str: str, out_tag: str) -> None: + input = make_generic_dataset(entity_urn=in_urn) + transformer = ExtractDatasetTags.create( + { + "extract_tags_from": "urn", + "extract_tags_regex": regex_str, + "semantics": "overwrite", + }, + PipelineContext(run_id="test"), + ) + output = list( + transformer.transform( + [ + RecordEnvelope(input, metadata={}), + RecordEnvelope(EndOfStream(), metadata={}), + ] + ) + ) + + assert len(output) == 3 + assert output[0].record == input + tags_aspect = output[1].record.aspect + assert isinstance(tags_aspect, GlobalTagsClass) + assert len(tags_aspect.tags) == 1 + assert tags_aspect.tags[0].tag == out_tag + + +def test_extract_dataset_tags(mock_time): + _test_extract_tags( + in_urn="urn:li:dataset:(urn:li:dataPlatform:kafka,clusterid.part1-part2-part3_part4,PROD)", + regex_str="(.*)", + out_tag="urn:li:tag:clusterid.part1-part2-part3_part4", + ) + _test_extract_tags( + in_urn="urn:li:dataset:(urn:li:dataPlatform:kafka,clusterid.USA-ops-team_table1,PROD)", + regex_str=".([^._]*)_", + out_tag="urn:li:tag:USA-ops-team", + ) + _test_extract_tags( + in_urn="urn:li:dataset:(urn:li:dataPlatform:kafka,clusterid.Canada-marketing_table1,PROD)", + regex_str=".([^._]*)_", + out_tag="urn:li:tag:Canada-marketing", + ) + _test_extract_tags( + in_urn="urn:li:dataset:(urn:li:dataPlatform:elasticsearch,abcdef-prefix_datahub_usage_event-000027,PROD)", + regex_str="([^._]*)_", + out_tag="urn:li:tag:abcdef-prefix", + ) + + def test_simple_dataset_ownership_with_invalid_type_transformation(mock_time): with pytest.raises(ValueError): SimpleAddDatasetOwnership.create( From 47616ff093b34960474949e6db0e62645330e846 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 20 Jul 2023 11:04:30 -0700 Subject: [PATCH 164/222] feat(ingest/lookml): fail when nothing was produced (#8464) --- .../src/datahub/ingestion/source/looker/lookml_source.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py index dedadb9bbecae..362b4e5530638 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py @@ -1853,6 +1853,13 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: yield from self.get_internal_workunits() + if not self.report.events_produced and not self.report.failures: + # Don't pass if we didn't produce any events. + self.report.report_failure( + "
", + "No metadata was produced. Check the logs for more details.", + ) + def _recursively_check_manifests( self, tmp_dir: str, project_name: str, project_visited: Set[str] ) -> None: @@ -1976,7 +1983,7 @@ def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 if connectionDefinition is None: self.reporter.report_warning( f"model-{model_name}", - f"Failed to load connection {model.connection}. Check your API key permissions.", + f"Failed to load connection {model.connection}. Check your API key permissions and/or connection_to_platform_map configuration.", ) self.reporter.report_models_dropped(model_name) continue From d733363bedf6d3558751e2ec28102f2468333620 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 20 Jul 2023 11:05:25 -0700 Subject: [PATCH 165/222] chore(ingest): drop bigquery-beta and snowflake-beta aliases (#8451) --- docs/how/updating-datahub.md | 1 + metadata-ingestion/setup.py | 11 ----------- .../ingestion/source/source_registry.py | 19 ------------------- 3 files changed, 1 insertion(+), 30 deletions(-) diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index b8ecd689ce381..03b3d763ed247 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -14,6 +14,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe `profile_table_level_only` together with `include_field_xyz` config options to ingest certain column-level metrics. Instead, set `profile_table_level_only` to `false` and individually enable / disable desired field metrics. +- #8451: The `bigquery-beta` and `snowflake-beta` source aliases have been dropped. Use `bigquery` and `snowflake` as the source type instead. ### Potential Downtime diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 36f4151df0359..04bc03a236f07 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -292,14 +292,6 @@ def get_long_description(): "sqlalchemy-bigquery>=1.4.1", "google-cloud-datacatalog-lineage==0.2.2", }, - "bigquery-beta": sql_common - | bigquery_common - | { - *sqllineage_lib, - *sqlglot_lib, - "sql_metadata", - "sqlalchemy-bigquery>=1.4.1", - }, # deprecated, but keeping the extra for backwards compatibility "clickhouse": sql_common | clickhouse_common, "clickhouse-usage": sql_common | usage_common | clickhouse_common, "datahub-lineage-file": set(), @@ -370,9 +362,6 @@ def get_long_description(): "sagemaker": aws_common, "salesforce": {"simple-salesforce"}, "snowflake": snowflake_common | usage_common | sqlglot_lib, - "snowflake-beta": ( - snowflake_common | usage_common | sqlglot_lib - ), # deprecated, but keeping the extra for backwards compatibility "sqlalchemy": sql_common, "superset": { "requests", diff --git a/metadata-ingestion/src/datahub/ingestion/source/source_registry.py b/metadata-ingestion/src/datahub/ingestion/source/source_registry.py index 23fe1bd3baf54..37f088bcd7b50 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/source_registry.py +++ b/metadata-ingestion/src/datahub/ingestion/source/source_registry.py @@ -8,25 +8,6 @@ source_registry.register_from_entrypoint("datahub.ingestion.source.plugins") # Deprecations. -source_registry.register_alias( - "snowflake-beta", - "snowflake", - lambda: warnings.warn( - "source type snowflake-beta is deprecated, use snowflake instead", - ConfigurationWarning, - stacklevel=3, - ), -) -source_registry.register_alias( - "bigquery-beta", - "bigquery", - lambda: warnings.warn( - "source type bigquery-beta is deprecated, use bigquery instead", - ConfigurationWarning, - stacklevel=3, - ), -) - source_registry.register_alias( "redshift-usage", "redshift-usage-legacy", From f4fde211686d86c2219a51ee8d856878cc16d992 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Thu, 20 Jul 2023 23:42:33 +0530 Subject: [PATCH 166/222] feat(ingest/nifi): add support for basic auth in nifi (#8457) --- metadata-ingestion/docs/sources/nifi/nifi.md | 3 + .../src/datahub/ingestion/source/nifi.py | 116 +++++++++++------- .../tests/integration/nifi/test_nifi.py | 66 +++++----- .../tests/unit/test_nifi_source.py | 18 ++- 4 files changed, 119 insertions(+), 84 deletions(-) diff --git a/metadata-ingestion/docs/sources/nifi/nifi.md b/metadata-ingestion/docs/sources/nifi/nifi.md index 5ea6a53186c0d..7da9430ee8ed5 100644 --- a/metadata-ingestion/docs/sources/nifi/nifi.md +++ b/metadata-ingestion/docs/sources/nifi/nifi.md @@ -15,6 +15,9 @@ If nifi has been configured to use [Kerberos SPNEGO](https://nifi.apache.org/doc sudo apt install krb5-user kinit user@REALM ``` +#### Basic Authentication (`auth: BASIC_AUTH`) +Connector will use [HTTPBasicAuth](https://requests.readthedocs.io/en/latest/user/authentication/#basic-authentication) with `username` and `password`. + #### No Authentication (`auth: NO_AUTH`) This is useful for testing purposes. diff --git a/metadata-ingestion/src/datahub/ingestion/source/nifi.py b/metadata-ingestion/src/datahub/ingestion/source/nifi.py index 27cd5aeb3c68d..559d103aa6e5a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/nifi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/nifi.py @@ -16,6 +16,7 @@ from pydantic.fields import Field from requests import Response from requests.adapters import HTTPAdapter +from requests.models import HTTPBasicAuth from requests_gssapi import HTTPSPNEGOAuth import datahub.emitter.mce_builder as builder @@ -66,6 +67,7 @@ class NifiAuthType(Enum): SINGLE_USER = "SINGLE_USER" CLIENT_CERT = "CLIENT_CERT" KERBEROS = "KERBEROS" + BASIC_AUTH = "BASIC_AUTH" class NifiSourceConfig(EnvConfigMixin): @@ -132,17 +134,17 @@ def validate_auth_params(cla, values): raise ValueError( "Config `client_cert_file` is required for CLIENT_CERT auth" ) - elif values.get("auth") is NifiAuthType.SINGLE_USER and ( - not values.get("username") or not values.get("password") - ): + elif values.get("auth") in ( + NifiAuthType.SINGLE_USER, + NifiAuthType.BASIC_AUTH, + ) and (not values.get("username") or not values.get("password")): raise ValueError( - "Config `username` and `password` is required for SINGLE_USER auth" + f"Config `username` and `password` is required for {values.get('auth').value} auth" ) return values @root_validator(pre=False) def validator_site_url_to_site_name(cls, values): - site_url_to_site_name = values.get("site_url_to_site_name") site_url = values.get("site_url") site_name = values.get("site_name") @@ -667,41 +669,8 @@ def fetch_provenance_events( of processor type {processor.type}, Start date: {startDate}, End date: {endDate}" ) - older_version: bool = self.nifi_flow.version is not None and version.parse( - self.nifi_flow.version - ) < version.parse("1.13.0") - - if older_version: - searchTerms = { - "ProcessorID": processor.id, - "EventType": eventType, - } - else: - searchTerms = { - "ProcessorID": {"value": processor.id}, # type: ignore - "EventType": {"value": eventType}, # type: ignore - } - - payload = json.dumps( - { - "provenance": { - "request": { - "maxResults": 1000, - "summarize": False, - "searchTerms": searchTerms, - "startDate": startDate.strftime("%m/%d/%Y %H:%M:%S %Z"), - "endDate": ( - endDate.strftime("%m/%d/%Y %H:%M:%S %Z") - if endDate - else None - ), - } - } - } - ) - logger.debug(payload) - provenance_response = self.session.post( - url=urljoin(self.rest_api_base_url, PROVENANCE_ENDPOINT), data=payload + provenance_response = self.submit_provenance_query( + processor, eventType, startDate, endDate ) if provenance_response.ok: @@ -759,6 +728,58 @@ def fetch_provenance_events( logger.warning(provenance_response.text) return + def submit_provenance_query(self, processor, eventType, startDate, endDate): + older_version: bool = self.nifi_flow.version is not None and version.parse( + self.nifi_flow.version + ) < version.parse("1.13.0") + + if older_version: + searchTerms = { + "ProcessorID": processor.id, + "EventType": eventType, + } + else: + searchTerms = { + "ProcessorID": {"value": processor.id}, # type: ignore + "EventType": {"value": eventType}, # type: ignore + } + + payload = json.dumps( + { + "provenance": { + "request": { + "maxResults": 1000, + "summarize": False, + "searchTerms": searchTerms, + "startDate": startDate.strftime("%m/%d/%Y %H:%M:%S %Z"), + "endDate": ( + endDate.strftime("%m/%d/%Y %H:%M:%S %Z") + if endDate + else None + ), + } + } + } + ) + logger.debug(payload) + self.session.headers.update({}) + + self.session.headers.update({"Content-Type": "application/json"}) + provenance_response = self.session.post( + url=urljoin(self.rest_api_base_url, PROVENANCE_ENDPOINT), + data=payload, + ) + + # Revert to default content-type if basic-auth + if self.config.auth is NifiAuthType.BASIC_AUTH: + self.session.headers.update( + { + "Content-Type": "application/x-www-form-urlencoded", + } + ) + + return provenance_response + def report_warning(self, key: str, reason: str) -> None: logger.warning(f"{key}: {reason}") self.report.report_warning(key, reason) @@ -958,6 +979,19 @@ def authenticate(self): # Token not required return + if self.config.auth is NifiAuthType.BASIC_AUTH: + assert self.config.username is not None + assert self.config.password is not None + self.session.auth = HTTPBasicAuth( + self.config.username, self.config.password + ) + self.session.headers.update( + { + "Content-Type": "application/x-www-form-urlencoded", + } + ) + return + if self.config.auth is NifiAuthType.CLIENT_CERT: self.session.mount( self.rest_api_base_url, @@ -1001,8 +1035,6 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: self.report.report_failure(self.config.site_url, "Failed to authenticate") return - self.session.headers.update({"Content-Type": "application/json"}) - # Creates nifi_flow by invoking /flow rest api and saves as self.nifi_flow try: self.create_nifi_flow() diff --git a/metadata-ingestion/tests/integration/nifi/test_nifi.py b/metadata-ingestion/tests/integration/nifi/test_nifi.py index 3ba194e9b5e12..58efd32c6deb3 100644 --- a/metadata-ingestion/tests/integration/nifi/test_nifi.py +++ b/metadata-ingestion/tests/integration/nifi/test_nifi.py @@ -107,7 +107,7 @@ def test_nifi_ingest_standalone( @freeze_time(FROZEN_TIME) @pytest.mark.slow_integration -def test_nifi_ingest_cluster(loaded_nifi, pytestconfig, test_resources_dir): +def test_nifi_ingest_cluster(loaded_nifi, pytestconfig, tmp_path, test_resources_dir): # Wait for nifi cluster to execute all lineage processors, max wait time 120 seconds url = "http://localhost:9080/nifi-api/flow/process-groups/root" for i in range(23): @@ -124,37 +124,39 @@ def test_nifi_ingest_cluster(loaded_nifi, pytestconfig, test_resources_dir): logging.info(f"Waited for time {i*5} seconds") break test_resources_dir = pytestconfig.rootpath / "tests/integration/nifi" - # Run nifi ingestion run. - pipeline = Pipeline.create( - { - "run_id": "nifi-test-cluster", - "source": { - "type": "nifi", - "config": { - "site_url": "http://localhost:9080/nifi/", - "auth": "NO_AUTH", - "site_url_to_site_name": { - "http://nifi01:9080/nifi/": "default", - "http://nifi02:9081/nifi/": "default", - "http://nifi03:9082/nifi/": "default", + # Run the metadata ingestion pipeline. + with fs_helpers.isolated_filesystem(tmp_path): + # Run nifi ingestion run. + pipeline = Pipeline.create( + { + "run_id": "nifi-test-cluster", + "source": { + "type": "nifi", + "config": { + "site_url": "http://localhost:9080/nifi/", + "auth": "NO_AUTH", + "site_url_to_site_name": { + "http://nifi01:9080/nifi/": "default", + "http://nifi02:9081/nifi/": "default", + "http://nifi03:9082/nifi/": "default", + }, }, }, - }, - "sink": { - "type": "file", - "config": {"filename": "./nifi_mces_cluster.json"}, - }, - } - ) - pipeline.run() - pipeline.raise_from_status() + "sink": { + "type": "file", + "config": {"filename": "./nifi_mces_cluster.json"}, + }, + } + ) + pipeline.run() + pipeline.raise_from_status() - # Verify the output. - mce_helpers.check_golden_file( - pytestconfig, - output_path="nifi_mces_cluster.json", - golden_path=test_resources_dir / "nifi_mces_golden_cluster.json", - ignore_paths=[ - r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['last_event_time'\]", - ], - ) + # Verify the output. + mce_helpers.check_golden_file( + pytestconfig, + output_path="nifi_mces_cluster.json", + golden_path=test_resources_dir / "nifi_mces_golden_cluster.json", + ignore_paths=[ + r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['last_event_time'\]", + ], + ) diff --git a/metadata-ingestion/tests/unit/test_nifi_source.py b/metadata-ingestion/tests/unit/test_nifi_source.py index a90f03bea3e42..d9e2b6e35e157 100644 --- a/metadata-ingestion/tests/unit/test_nifi_source.py +++ b/metadata-ingestion/tests/unit/test_nifi_source.py @@ -278,27 +278,29 @@ def mocked_functions(mock_provenance_events, mock_delete_provenance, provenance_ mock_provenance_events.return_value = puts3_provenance_response -def test_single_user_auth_without_password(): +@pytest.mark.parametrize("auth", ["SINGLE_USER", "BASIC_AUTH"]) +def test_auth_without_password(auth): with pytest.raises( - ValueError, match="`username` and `password` is required for SINGLE_USER auth" + ValueError, match=f"`username` and `password` is required for {auth} auth" ): NifiSourceConfig.parse_obj( { "site_url": "https://localhost:8443", - "auth": "SINGLE_USER", + "auth": auth, "username": "someuser", } ) -def test_single_user_auth_without_username_and_password(): +@pytest.mark.parametrize("auth", ["SINGLE_USER", "BASIC_AUTH"]) +def test_auth_without_username_and_password(auth): with pytest.raises( - ValueError, match="`username` and `password` is required for SINGLE_USER auth" + ValueError, match=f"`username` and `password` is required for {auth} auth" ): NifiSourceConfig.parse_obj( { "site_url": "https://localhost:8443", - "auth": "SINGLE_USER", + "auth": auth, } ) @@ -316,7 +318,6 @@ def test_client_cert_auth_without_client_cert_file(): def test_single_user_auth_failed_to_get_token(): - config = NifiSourceConfig( site_url="https://localhost:12345", # will never work username="username", @@ -338,7 +339,6 @@ def test_single_user_auth_failed_to_get_token(): def test_kerberos_auth_failed_to_get_token(): - config = NifiSourceConfig( site_url="https://localhost:12345", # will never work auth="KERBEROS", @@ -358,7 +358,6 @@ def test_kerberos_auth_failed_to_get_token(): def test_client_cert_auth_failed(): - config = NifiSourceConfig( site_url="https://localhost:12345", # will never work auth="CLIENT_CERT", @@ -379,7 +378,6 @@ def test_client_cert_auth_failed(): def test_failure_to_create_nifi_flow(): - with patch("datahub.ingestion.source.nifi.NifiSource.authenticate"): config = NifiSourceConfig( site_url="https://localhost:12345", # will never work From b5e039ff4e083382134295b9fd1dcbb88d79d83c Mon Sep 17 00:00:00 2001 From: Kos Korchak <97058061+kkorchak@users.noreply.github.com> Date: Thu, 20 Jul 2023 15:54:33 -0400 Subject: [PATCH 167/222] fix(tests): Fix query_tab test that was failing on CI run (#8463) --- .../cypress/cypress/e2e/query/query_tab.js | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/smoke-test/tests/cypress/cypress/e2e/query/query_tab.js b/smoke-test/tests/cypress/cypress/e2e/query/query_tab.js index 0d8bcf7b4a655..4d01cac15724e 100644 --- a/smoke-test/tests/cypress/cypress/e2e/query/query_tab.js +++ b/smoke-test/tests/cypress/cypress/e2e/query/query_tab.js @@ -11,7 +11,7 @@ describe("manage queries", () => { cy.openEntityTab("Queries") }) - it("go to querys tab on dataset page then, create, edit, make default, delete a view", () => { + it("go to queries tab on dataset page then, create, edit, make default, delete a view", () => { const runId = Date.now() // Headers @@ -27,6 +27,7 @@ describe("manage queries", () => { cy.get('.ProseMirror').click(); cy.get('.ProseMirror').type(`Test Description-${runId}`); cy.get('[data-testid="query-builder-save-button"]').click(); + cy.waitTextVisible("Created Query!"); // Verify the card cy.waitTextVisible(`+ Test Query-${runId}`); @@ -36,10 +37,10 @@ describe("manage queries", () => { // View the Query cy.get('[data-testid="query-content-0"]').click(); + cy.get('.ant-modal-content').waitTextVisible(`+ Test Query-${runId}`); + cy.get('.ant-modal-content').waitTextVisible(`Test Table-${runId}`); + cy.get('.ant-modal-content').waitTextVisible(`Test Description-${runId}`); cy.get('[data-testid="query-modal-close-button"]').click(); - cy.waitTextVisible(`+ Test Query-${runId}`); - cy.waitTextVisible(`Test Table-${runId}`); - cy.waitTextVisible(`Test Description-${runId}`); // Edit the Query cy.get('[data-testid="query-edit-button-0"]').click() @@ -52,20 +53,23 @@ describe("manage queries", () => { cy.get('.ProseMirror').clear(); cy.get('.ProseMirror').type(`Edited Description-${runId}`); cy.get('[data-testid="query-builder-save-button"]').click(); + cy.waitTextVisible("Edited Query!"); - // Verify the card + // Verify edited Query card + cy.get('[data-testid="query-content-0"]').scrollIntoView().should('be.visible'); cy.waitTextVisible(`+ Test Query-${runId} + Edited Query-${runId}`); - cy.waitTextVisible(`Edited Description-${runId}`); + cy.waitTextVisible(`Edited Table-${runId}`); cy.waitTextVisible(`Edited Description-${runId}`); // Delete the Query cy.get('[data-testid="query-more-button-0"]').click(); cy.get('[data-testid="query-delete-button-0"]').click(); cy.contains('Yes').click(); + cy.waitTextVisible("Deleted Query!"); // Query should be gone cy.ensureTextNotPresent(`+ Test Query-${runId} + Edited Query-${runId}`); - cy.ensureTextNotPresent(`Edited Description-${runId}`); + cy.ensureTextNotPresent(`Edited Table-${runId}`); cy.ensureTextNotPresent(`Edited Description-${runId}`); }); }); \ No newline at end of file From f4c0ed3aab445b05ecaccd5f3c68c3861f30444c Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Fri, 21 Jul 2023 03:01:06 +0530 Subject: [PATCH 168/222] ingest(mysql): add storage bytes information (#8294) Co-authored-by: Andrew Sikowitz --- .../src/datahub/ingestion/source/sql/mysql.py | 12 +++++++ .../ingestion/source/sql/sql_common.py | 36 +++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mysql.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mysql.py index 3be9a5df4f0b9..3e89dd53c1eec 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/mysql.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mysql.py @@ -5,6 +5,7 @@ from sqlalchemy import util from sqlalchemy.dialects.mysql import base from sqlalchemy.dialects.mysql.enumerated import SET +from sqlalchemy.engine.reflection import Inspector from datahub.ingestion.api.decorators import ( SourceCapability, @@ -83,3 +84,14 @@ def get_platform(self): def create(cls, config_dict, ctx): config = MySQLConfig.parse_obj(config_dict) return cls(config, ctx) + + def add_profile_metadata(self, inspector: Inspector) -> None: + if not self.config.profiling.enabled: + return + with inspector.engine.connect() as conn: + for row in conn.execute( + "SELECT table_schema, table_name, data_length from information_schema.tables" + ): + self.profile_metadata_info.dataset_name_to_storage_bytes[ + f"{row.table_schema}.{row.table_name}" + ] = row.data_length diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py index fb659d9548540..42ea7aed9b620 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py @@ -309,6 +309,15 @@ def get_schema_metadata( ] +@dataclass +class ProfileMetadata: + """ + A class to hold information about the table for profile enrichment + """ + + dataset_name_to_storage_bytes: Dict[str, int] = field(default_factory=dict) + + class SQLAlchemySource(StatefulIngestionSourceBase): """A Base class for all SQL Sources that use SQLAlchemy to extend""" @@ -317,6 +326,7 @@ def __init__(self, config: SQLAlchemyConfig, ctx: PipelineContext, platform: str self.config = config self.platform = platform self.report: SQLSourceReport = SQLSourceReport() + self.profile_metadata_info: ProfileMetadata = ProfileMetadata() config_report = { config_option: config.dict().get(config_option) @@ -484,6 +494,16 @@ def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit profile_requests: List["GEProfilerRequest"] = [] if sql_config.profiling.enabled: profiler = self.get_profiler_instance(inspector) + try: + self.add_profile_metadata(inspector) + except Exception as e: + logger.warning( + "Failed to get enrichment data for profiler", exc_info=True + ) + self.report.report_warning( + "profile_metadata", + f"Failed to get enrichment data for profile {e}", + ) db_name = self.get_db_name(inspector) yield from self.gen_database_containers( @@ -1098,6 +1118,13 @@ def loop_profiler_requests( ), ) + def add_profile_metadata(self, inspector: Inspector) -> None: + """ + Method to add profile metadata in a sub-class that can be used to enrich profile metadata. + This is meant to change self.profile_metadata_info in the sub-class. + """ + pass + def loop_profiler( self, profile_requests: List["GEProfilerRequest"], @@ -1113,6 +1140,15 @@ def loop_profiler( if profile is None: continue dataset_name = request.pretty_name + if ( + dataset_name in self.profile_metadata_info.dataset_name_to_storage_bytes + and profile.sizeInBytes is None + ): + profile.sizeInBytes = ( + self.profile_metadata_info.dataset_name_to_storage_bytes[ + dataset_name + ] + ) dataset_urn = make_dataset_urn_with_platform_instance( self.platform, dataset_name, From a4f4ff386c4dbea1ec51798a972f5eb4f3ef80b7 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Fri, 21 Jul 2023 10:44:57 -0400 Subject: [PATCH 169/222] fix(cache) Fix caching bug with new search filters (#8434) Co-authored-by: Aseem Bansal --- .../metadata/search/client/CachingEntitySearchService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java index 804ab3303f6b0..56f6fed3ad9d2 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java @@ -154,8 +154,8 @@ public SearchResult getCachedSearchResults( batchSize, querySize -> getRawSearchResults(entityName, query, filters, sortCriterion, querySize.getFrom(), querySize.getSize(), flags, facets), - querySize -> Quintet.with(entityName, query, filters != null ? toJsonString(filters) : null, - sortCriterion != null ? toJsonString(sortCriterion) : null, querySize), flags, enableCache).getSearchResults(from, size); + querySize -> Sextet.with(entityName, query, filters != null ? toJsonString(filters) : null, + sortCriterion != null ? toJsonString(sortCriterion) : null, facets, querySize), flags, enableCache).getSearchResults(from, size); } From 392f9f1bc018f1f9f767273a74458e986885a843 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Fri, 21 Jul 2023 10:48:46 -0400 Subject: [PATCH 170/222] fix(browseV2) Escape forward slashes in browse v2 query (#8446) --- .../resolvers/chart/BrowseV2Resolver.java | 4 +++- .../src/app/home/AcrylDemoBanner.tsx | 21 +++++++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java index 41a1d22485ea4..76abddc9a99a9 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java @@ -49,6 +49,8 @@ public CompletableFuture get(DataFetchingEnvironment environmen final int start = input.getStart() != null ? input.getStart() : DEFAULT_START; final int count = input.getCount() != null ? input.getCount() : DEFAULT_COUNT; final String query = input.getQuery() != null ? input.getQuery() : "*"; + // escape forward slash since it is a reserved character in Elasticsearch + final String sanitizedQuery = ResolverUtils.escapeForwardSlash(query); return CompletableFuture.supplyAsync(() -> { try { @@ -64,7 +66,7 @@ public CompletableFuture get(DataFetchingEnvironment environmen maybeResolvedView != null ? SearchUtils.combineFilters(filter, maybeResolvedView.getDefinition().getFilter()) : filter, - query, + sanitizedQuery, start, count, context.getAuthentication() diff --git a/datahub-web-react/src/app/home/AcrylDemoBanner.tsx b/datahub-web-react/src/app/home/AcrylDemoBanner.tsx index 87efae03e0a7c..0a6316a71db16 100644 --- a/datahub-web-react/src/app/home/AcrylDemoBanner.tsx +++ b/datahub-web-react/src/app/home/AcrylDemoBanner.tsx @@ -33,13 +33,17 @@ const StyledLink = styled(Link)` font-weight: 700; `; +const TextContent = styled.div` + max-width: 1025px; +`; + export default function AcrylDemoBanner() { return ( - Schedule a Demo of Managed Datahub - + Schedule a Demo of Managed DataHub + DataHub is already the industry's #1 Open Source Data Catalog.{' '} Schedule a demo {' '} - of Acryl Cloud to see the advanced features that take it to the next level! - + of Acryl DataHub to see the advanced features that take it to the next level or purchase Acryl Cloud + on{' '} + + AWS Marketplace + + ! + ); From bec018257c8c7c9af7dc79d91e8db34414188ff0 Mon Sep 17 00:00:00 2001 From: mohdsiddique Date: Sat, 22 Jul 2023 02:31:57 +0530 Subject: [PATCH 171/222] fix(ingestion/powerbi-report-srever): handle requests.exceptions.JSONDecodeError (#8442) Co-authored-by: MohdSiddiqueBagwan --- .../source/powerbi_report_server/constants.py | 2 - .../powerbi_report_server/report_server.py | 49 ++- .../report_server_domain.py | 5 - .../golden_test_fail_api_ingest.json | 326 +++++++++++++++++ .../golden_test_ingest.json | 334 +++++++++++------- .../test_powerbi_report_server.py | 123 ++++--- 6 files changed, 641 insertions(+), 198 deletions(-) create mode 100644 metadata-ingestion/tests/integration/powerbi_report_server/golden_test_fail_api_ingest.json diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/constants.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/constants.py index 92560a11b90eb..9f409793272dd 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/constants.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/constants.py @@ -82,8 +82,6 @@ class Constant: Constant.LINKED_REPORTS: "{PBIRS_BASE_URL}/LinkedReports", Constant.LINKED_REPORT: "{PBIRS_BASE_URL}/LinkedReports({LINKED_REPORT_ID})", Constant.ME: "{PBIRS_BASE_URLL}/Me", - Constant.MOBILE_REPORTS: "{PBIRS_BASE_URL}/MobileReports", - Constant.MOBILE_REPORT: "{PBIRS_BASE_URL}/MobileReports({MOBILE_REPORT_ID})", Constant.POWERBI_REPORTS: "{PBIRS_BASE_URL}/PowerBiReports", Constant.POWERBI_REPORT: "{PBIRS_BASE_URL}/PowerBiReports({POWERBI_REPORT_ID})", Constant.POWERBI_REPORT_DATASOURCES: "{PBIRS_BASE_URL}/PowerBiReports({ID})/DataSources", diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py index 7419241511eaf..e66119f6e8d76 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py @@ -35,7 +35,6 @@ from datahub.ingestion.source.powerbi_report_server.report_server_domain import ( CorpUser, LinkedReport, - MobileReport, Owner, OwnershipData, PowerBiReport, @@ -116,6 +115,29 @@ class PowerBiReportServerDashboardSourceConfig(PowerBiReportServerAPIConfig): chart_pattern: AllowDenyPattern = AllowDenyPattern.allow_all() +def log_http_error(e: BaseException, message: str) -> Any: + LOGGER.warning(message) + + if isinstance(e, requests.exceptions.HTTPError): + LOGGER.warning(f"HTTP status-code = {e.response.status_code}") + + LOGGER.debug(msg=message, exc_info=e) + + return e + + +def get_response_dict(response: requests.Response, error_message: str) -> dict: + + result_dict: dict = {} + try: + response.raise_for_status() + result_dict = response.json() + except BaseException as e: + log_http_error(e=e, message=error_message) + + return result_dict + + class PowerBiReportServerAPI: # API endpoints of PowerBI Report Server to fetch reports, datasets @@ -144,14 +166,15 @@ def requests_get(self, url_http: str, url_https: str, content_type: str) -> Any: url=url_http, auth=self.get_auth_credentials, ) - # Check if we got response from PowerBi Report Server - if response.status_code != 200: - message: str = "Failed to fetch Report from powerbi-report-server for" - LOGGER.warning(message) - LOGGER.warning("{}={}".format(Constant.ReportId, content_type)) - raise ValueError(message) - return response.json() + error_message: str = ( + f"Failed to fetch {content_type} Report from powerbi-report-server" + ) + + return get_response_dict( + response=response, + error_message=error_message, + ) def get_all_reports(self) -> List[Any]: """ @@ -159,7 +182,6 @@ def get_all_reports(self) -> List[Any]: """ report_types_mapping: Dict[str, Any] = { Constant.REPORTS: Report, - Constant.MOBILE_REPORTS: MobileReport, Constant.LINKED_REPORTS: LinkedReport, Constant.POWERBI_REPORTS: PowerBiReport, } @@ -174,15 +196,17 @@ def get_all_reports(self) -> List[Any]: report_get_endpoint_https = report_get_endpoint.format( PBIRS_BASE_URL=self.__config.get_base_api_https_url, ) + response_dict = self.requests_get( url_http=report_get_endpoint_http, url_https=report_get_endpoint_https, content_type=report_type, - )["value"] - if response_dict: + ) + + if response_dict.get("value"): reports.extend( report_types_mapping[report_type].parse_obj(report) - for report in response_dict + for report in response_dict.get("value") ) return reports @@ -487,7 +511,6 @@ class PowerBiReportServerDashboardSource(Source): Next types of report can be ingested: - PowerBI report(.pbix) - Paginated report(.rdl) - - Mobile report - Linked report """ diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server_domain.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server_domain.py index adcbcaaed96e6..60426fc5bd660 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server_domain.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server_domain.py @@ -239,11 +239,6 @@ class Manifest(BaseModel): resources: List[Dict[str, List]] = Field(alias="Resources") -class MobileReport(CatalogItem): - allow_caching: bool = Field(alias="AllowCaching") - manifest: Manifest = Field(alias="Manifest") - - class PowerBIReport(CatalogItem): has_data_sources: bool = Field(alias="HasDataSources") diff --git a/metadata-ingestion/tests/integration/powerbi_report_server/golden_test_fail_api_ingest.json b/metadata-ingestion/tests/integration/powerbi_report_server/golden_test_fail_api_ingest.json new file mode 100644 index 0000000000000..69a567654cac1 --- /dev/null +++ b/metadata-ingestion/tests/integration/powerbi_report_server/golden_test_fail_api_ingest.json @@ -0,0 +1,326 @@ +[ +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:TEST_USER", + "changeType": "UPSERT", + "aspectName": "corpUserInfo", + "aspect": { + "json": { + "customProperties": {}, + "active": true, + "displayName": "TEST_USER", + "email": "" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-report-server-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:TEST_USER", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-report-server-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:TEST_USER", + "changeType": "UPSERT", + "aspectName": "corpUserKey", + "aspect": { + "json": { + "username": "TEST_USER" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-report-server-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938a)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "json": { + "paths": [ + "/powerbi_report_server/dev/server_alias/Reports/path/to/Testa" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-report-server-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938a)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "workspaceName": "PowerBI Report Server", + "workspaceId": "host_port", + "createdBy": "TEST_USER", + "createdDate": "2022-02-03 07:00:00", + "modifiedBy": "TEST_USER", + "modifiedDate": "2022-02-03 07:00:00", + "dataSource": "" + }, + "title": "Testa", + "description": "", + "charts": [], + "datasets": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "dashboardUrl": "http://host_port/Reports/powerbi/path/to/Testa" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-report-server-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938a)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-report-server-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938a)", + "changeType": "UPSERT", + "aspectName": "dashboardKey", + "aspect": { + "json": { + "dashboardTool": "powerbi", + "dashboardId": "powerbi.linkedin.com/dashboards/ee56dc21-248a-4138-a446-ee5ab1fc938a" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-report-server-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938a)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:TEST_USER", + "type": "TECHNICAL_OWNER" + }, + { + "owner": "urn:li:corpuser:TEST_USER", + "type": "NONE" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-report-server-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:TEST_USER", + "changeType": "UPSERT", + "aspectName": "corpUserInfo", + "aspect": { + "json": { + "customProperties": {}, + "active": true, + "displayName": "TEST_USER", + "email": "" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-report-server-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:TEST_USER", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-report-server-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:TEST_USER", + "changeType": "UPSERT", + "aspectName": "corpUserKey", + "aspect": { + "json": { + "username": "TEST_USER" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-report-server-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938d)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "json": { + "paths": [ + "/powerbi_report_server/dev/server_alias/Reports/path/to/Testd" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-report-server-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938d)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "workspaceName": "PowerBI Report Server", + "workspaceId": "host_port", + "createdBy": "TEST_USER", + "createdDate": "2022-02-03 07:00:00", + "modifiedBy": "TEST_USER", + "modifiedDate": "2022-02-03 07:00:00", + "dataSource": "" + }, + "title": "Testd", + "description": "", + "charts": [], + "datasets": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "dashboardUrl": "http://host_port/Reports/powerbi/path/to/Testd" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-report-server-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938d)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-report-server-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938d)", + "changeType": "UPSERT", + "aspectName": "dashboardKey", + "aspect": { + "json": { + "dashboardTool": "powerbi", + "dashboardId": "powerbi.linkedin.com/dashboards/ee56dc21-248a-4138-a446-ee5ab1fc938d" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-report-server-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938d)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:TEST_USER", + "type": "TECHNICAL_OWNER" + }, + { + "owner": "urn:li:corpuser:TEST_USER", + "type": "NONE" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-report-server-test" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi_report_server/golden_test_ingest.json b/metadata-ingestion/tests/integration/powerbi_report_server/golden_test_ingest.json index 9b202baa947d5..f4277e41d58c7 100644 --- a/metadata-ingestion/tests/integration/powerbi_report_server/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi_report_server/golden_test_ingest.json @@ -5,8 +5,12 @@ "changeType": "UPSERT", "aspectName": "corpUserInfo", "aspect": { - "value": "{\"active\": true, \"displayName\": \"TEST_USER\", \"email\": \"\"}", - "contentType": "application/json" + "json": { + "customProperties": {}, + "active": true, + "displayName": "TEST_USER", + "email": "" + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -19,8 +23,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -33,8 +38,9 @@ "changeType": "UPSERT", "aspectName": "corpUserKey", "aspect": { - "value": "{\"username\": \"TEST_USER\"}", - "contentType": "application/json" + "json": { + "username": "TEST_USER" + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -47,8 +53,11 @@ "changeType": "UPSERT", "aspectName": "browsePaths", "aspect": { - "value": "{\"paths\": [\"/powerbi_report_server/dev/server_alias/Reports/path/to/Testa\"]}", - "contentType": "application/json" + "json": { + "paths": [ + "/powerbi_report_server/dev/server_alias/Reports/path/to/Testa" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -61,8 +70,32 @@ "changeType": "UPSERT", "aspectName": "dashboardInfo", "aspect": { - "value": "{\"customProperties\": {\"workspaceName\": \"PowerBI Report Server\", \"workspaceId\": \"host_port\", \"createdBy\": \"TEST_USER\", \"createdDate\": \"2022-02-03 07:00:00\", \"modifiedBy\": \"TEST_USER\", \"modifiedDate\": \"2022-02-03 07:00:00\", \"dataSource\": \"\"}, \"title\": \"Testa\", \"description\": \"\", \"charts\": [], \"datasets\": [], \"lastModified\": {\"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}, \"dashboardUrl\": \"http://host_port/Reports/powerbi/path/to/Testa\"}", - "contentType": "application/json" + "json": { + "customProperties": { + "workspaceName": "PowerBI Report Server", + "workspaceId": "host_port", + "createdBy": "TEST_USER", + "createdDate": "2022-02-03 07:00:00", + "modifiedBy": "TEST_USER", + "modifiedDate": "2022-02-03 07:00:00", + "dataSource": "" + }, + "title": "Testa", + "description": "", + "charts": [], + "datasets": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "dashboardUrl": "http://host_port/Reports/powerbi/path/to/Testa" + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -75,8 +108,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -89,8 +123,10 @@ "changeType": "UPSERT", "aspectName": "dashboardKey", "aspect": { - "value": "{\"dashboardTool\": \"powerbi\", \"dashboardId\": \"powerbi.linkedin.com/dashboards/ee56dc21-248a-4138-a446-ee5ab1fc938a\"}", - "contentType": "application/json" + "json": { + "dashboardTool": "powerbi", + "dashboardId": "powerbi.linkedin.com/dashboards/ee56dc21-248a-4138-a446-ee5ab1fc938a" + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -103,8 +139,22 @@ "changeType": "UPSERT", "aspectName": "ownership", "aspect": { - "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:TEST_USER\", \"type\": \"TECHNICAL_OWNER\"}, {\"owner\": \"urn:li:corpuser:TEST_USER\", \"type\": \"NONE\"}], \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}", - "contentType": "application/json" + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:TEST_USER", + "type": "TECHNICAL_OWNER" + }, + { + "owner": "urn:li:corpuser:TEST_USER", + "type": "NONE" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -117,8 +167,12 @@ "changeType": "UPSERT", "aspectName": "corpUserInfo", "aspect": { - "value": "{\"active\": true, \"displayName\": \"TEST_USER\", \"email\": \"\"}", - "contentType": "application/json" + "json": { + "customProperties": {}, + "active": true, + "displayName": "TEST_USER", + "email": "" + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -131,8 +185,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -145,78 +200,9 @@ "changeType": "UPSERT", "aspectName": "corpUserKey", "aspect": { - "value": "{\"username\": \"TEST_USER\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-report-server-test" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938b)", - "changeType": "UPSERT", - "aspectName": "browsePaths", - "aspect": { - "value": "{\"paths\": [\"/powerbi_report_server/dev/server_alias/Reports/path/to/Testb\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-report-server-test" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938b)", - "changeType": "UPSERT", - "aspectName": "dashboardInfo", - "aspect": { - "value": "{\"customProperties\": {\"workspaceName\": \"PowerBI Report Server\", \"workspaceId\": \"host_port\", \"createdBy\": \"TEST_USER\", \"createdDate\": \"2022-02-03 07:00:00\", \"modifiedBy\": \"TEST_USER\", \"modifiedDate\": \"2022-02-03 07:00:00\", \"dataSource\": \"\"}, \"title\": \"Testb\", \"description\": \"\", \"charts\": [], \"datasets\": [], \"lastModified\": {\"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}, \"dashboardUrl\": \"http://host_port/Reports/powerbi/path/to/Testb\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-report-server-test" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938b)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-report-server-test" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938b)", - "changeType": "UPSERT", - "aspectName": "dashboardKey", - "aspect": { - "value": "{\"dashboardTool\": \"powerbi\", \"dashboardId\": \"powerbi.linkedin.com/dashboards/ee56dc21-248a-4138-a446-ee5ab1fc938b\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-report-server-test" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938b)", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:TEST_USER\", \"type\": \"TECHNICAL_OWNER\"}, {\"owner\": \"urn:li:corpuser:TEST_USER\", \"type\": \"NONE\"}], \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}", - "contentType": "application/json" + "json": { + "username": "TEST_USER" + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -229,8 +215,12 @@ "changeType": "UPSERT", "aspectName": "corpUserInfo", "aspect": { - "value": "{\"active\": true, \"displayName\": \"TEST_USER\", \"email\": \"\"}", - "contentType": "application/json" + "json": { + "customProperties": {}, + "active": true, + "displayName": "TEST_USER", + "email": "" + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -243,8 +233,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -257,8 +248,9 @@ "changeType": "UPSERT", "aspectName": "corpUserKey", "aspect": { - "value": "{\"username\": \"TEST_USER\"}", - "contentType": "application/json" + "json": { + "username": "TEST_USER" + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -271,8 +263,11 @@ "changeType": "UPSERT", "aspectName": "browsePaths", "aspect": { - "value": "{\"paths\": [\"/powerbi_report_server/dev/server_alias/Reports/path/to/Testc\"]}", - "contentType": "application/json" + "json": { + "paths": [ + "/powerbi_report_server/dev/server_alias/Reports/path/to/Testc" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -285,8 +280,32 @@ "changeType": "UPSERT", "aspectName": "dashboardInfo", "aspect": { - "value": "{\"customProperties\": {\"workspaceName\": \"PowerBI Report Server\", \"workspaceId\": \"host_port\", \"createdBy\": \"TEST_USER\", \"createdDate\": \"2022-02-03 07:00:00\", \"modifiedBy\": \"TEST_USER\", \"modifiedDate\": \"2022-02-03 07:00:00\", \"dataSource\": \"\"}, \"title\": \"Testc\", \"description\": \"\", \"charts\": [], \"datasets\": [], \"lastModified\": {\"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}, \"dashboardUrl\": \"http://host_port/Reports/powerbi/path/to/Testc\"}", - "contentType": "application/json" + "json": { + "customProperties": { + "workspaceName": "PowerBI Report Server", + "workspaceId": "host_port", + "createdBy": "TEST_USER", + "createdDate": "2022-02-03 07:00:00", + "modifiedBy": "TEST_USER", + "modifiedDate": "2022-02-03 07:00:00", + "dataSource": "" + }, + "title": "Testc", + "description": "", + "charts": [], + "datasets": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "dashboardUrl": "http://host_port/Reports/powerbi/path/to/Testc" + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -299,8 +318,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -313,8 +333,10 @@ "changeType": "UPSERT", "aspectName": "dashboardKey", "aspect": { - "value": "{\"dashboardTool\": \"powerbi\", \"dashboardId\": \"powerbi.linkedin.com/dashboards/ee56dc21-248a-4138-a446-ee5ab1fc938c\"}", - "contentType": "application/json" + "json": { + "dashboardTool": "powerbi", + "dashboardId": "powerbi.linkedin.com/dashboards/ee56dc21-248a-4138-a446-ee5ab1fc938c" + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -327,8 +349,22 @@ "changeType": "UPSERT", "aspectName": "ownership", "aspect": { - "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:TEST_USER\", \"type\": \"TECHNICAL_OWNER\"}, {\"owner\": \"urn:li:corpuser:TEST_USER\", \"type\": \"NONE\"}], \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}", - "contentType": "application/json" + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:TEST_USER", + "type": "TECHNICAL_OWNER" + }, + { + "owner": "urn:li:corpuser:TEST_USER", + "type": "NONE" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -341,8 +377,12 @@ "changeType": "UPSERT", "aspectName": "corpUserInfo", "aspect": { - "value": "{\"active\": true, \"displayName\": \"TEST_USER\", \"email\": \"\"}", - "contentType": "application/json" + "json": { + "customProperties": {}, + "active": true, + "displayName": "TEST_USER", + "email": "" + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -355,8 +395,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -369,8 +410,9 @@ "changeType": "UPSERT", "aspectName": "corpUserKey", "aspect": { - "value": "{\"username\": \"TEST_USER\"}", - "contentType": "application/json" + "json": { + "username": "TEST_USER" + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -383,8 +425,11 @@ "changeType": "UPSERT", "aspectName": "browsePaths", "aspect": { - "value": "{\"paths\": [\"/powerbi_report_server/dev/server_alias/Reports/path/to/Testd\"]}", - "contentType": "application/json" + "json": { + "paths": [ + "/powerbi_report_server/dev/server_alias/Reports/path/to/Testd" + ] + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -397,8 +442,32 @@ "changeType": "UPSERT", "aspectName": "dashboardInfo", "aspect": { - "value": "{\"customProperties\": {\"workspaceName\": \"PowerBI Report Server\", \"workspaceId\": \"host_port\", \"createdBy\": \"TEST_USER\", \"createdDate\": \"2022-02-03 07:00:00\", \"modifiedBy\": \"TEST_USER\", \"modifiedDate\": \"2022-02-03 07:00:00\", \"dataSource\": \"\"}, \"title\": \"Testd\", \"description\": \"\", \"charts\": [], \"datasets\": [], \"lastModified\": {\"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}, \"dashboardUrl\": \"http://host_port/Reports/powerbi/path/to/Testd\"}", - "contentType": "application/json" + "json": { + "customProperties": { + "workspaceName": "PowerBI Report Server", + "workspaceId": "host_port", + "createdBy": "TEST_USER", + "createdDate": "2022-02-03 07:00:00", + "modifiedBy": "TEST_USER", + "modifiedDate": "2022-02-03 07:00:00", + "dataSource": "" + }, + "title": "Testd", + "description": "", + "charts": [], + "datasets": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "dashboardUrl": "http://host_port/Reports/powerbi/path/to/Testd" + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -411,8 +480,9 @@ "changeType": "UPSERT", "aspectName": "status", "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -425,8 +495,10 @@ "changeType": "UPSERT", "aspectName": "dashboardKey", "aspect": { - "value": "{\"dashboardTool\": \"powerbi\", \"dashboardId\": \"powerbi.linkedin.com/dashboards/ee56dc21-248a-4138-a446-ee5ab1fc938d\"}", - "contentType": "application/json" + "json": { + "dashboardTool": "powerbi", + "dashboardId": "powerbi.linkedin.com/dashboards/ee56dc21-248a-4138-a446-ee5ab1fc938d" + } }, "systemMetadata": { "lastObserved": 1643871600000, @@ -439,8 +511,22 @@ "changeType": "UPSERT", "aspectName": "ownership", "aspect": { - "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:TEST_USER\", \"type\": \"TECHNICAL_OWNER\"}, {\"owner\": \"urn:li:corpuser:TEST_USER\", \"type\": \"NONE\"}], \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}", - "contentType": "application/json" + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:TEST_USER", + "type": "TECHNICAL_OWNER" + }, + { + "owner": "urn:li:corpuser:TEST_USER", + "type": "NONE" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } }, "systemMetadata": { "lastObserved": 1643871600000, diff --git a/metadata-ingestion/tests/integration/powerbi_report_server/test_powerbi_report_server.py b/metadata-ingestion/tests/integration/powerbi_report_server/test_powerbi_report_server.py index c03190be66964..826c2b77bce36 100644 --- a/metadata-ingestion/tests/integration/powerbi_report_server/test_powerbi_report_server.py +++ b/metadata-ingestion/tests/integration/powerbi_report_server/test_powerbi_report_server.py @@ -21,7 +21,7 @@ def mock_user_to_add(*args, **kwargs): return None -def register_mock_api(request_mock): +def register_mock_api(request_mock, override_mock_data={}): api_vs_response = { "https://host_port/Reports/api/v2.0/Reports": { "method": "GET", @@ -52,37 +52,6 @@ def register_mock_api(request_mock): ] }, }, - "https://host_port/Reports/api/v2.0/MobileReports": { - "method": "GET", - "status_code": 200, - "json": { - "value": [ - { - "Id": "ee56dc21-248a-4138-a446-ee5ab1fc938b", - "Name": "Testb", - "Description": None, - "Path": "/path/to/Testb", - "Type": "MobileReport", - "Hidden": False, - "Size": 1010101, - "ModifiedBy": "TEST_USER", - "ModifiedDate": str(datetime.now()), - "CreatedBy": "TEST_USER", - "CreatedDate": str(datetime.now()), - "ParentFolderId": "47495172-89ab-455f-a446-fffd3cf239cb", - "IsFavorite": False, - "ContentType": None, - "Content": "", - "HasDataSources": True, - "Roles": [], - "HasSharedDataSets": True, - "HasParameters": True, - "AllowCaching": True, - "Manifest": {"Resources": []}, - }, - ] - }, - }, "https://host_port/Reports/api/v2.0/LinkedReports": { "method": "GET", "status_code": 200, @@ -141,6 +110,8 @@ def register_mock_api(request_mock): }, } + api_vs_response.update(override_mock_data) + for url in api_vs_response.keys(): request_mock.register_uri( api_vs_response[url]["method"], @@ -164,6 +135,30 @@ def default_source_config(): } +def get_default_recipe(output_path: str) -> dict: + return { + "run_id": "powerbi-report-server-test", + "source": { + "type": "powerbi-report-server", + "config": { + **default_source_config(), + }, + }, + "sink": { + "type": "file", + "config": {"filename": output_path}, # , + }, + } + + +def add_mock_method_in_pipeline(pipeline: Pipeline) -> None: + pipeline.ctx.graph = mock.MagicMock() + pipeline.ctx.graph.get_ownership = mock.MagicMock() + pipeline.ctx.graph.get_ownership.side_effect = mock_existing_users + pipeline.ctx.graph.get_aspect_v2 = mock.MagicMock() + pipeline.ctx.graph.get_aspect_v2.side_effect = mock_user_to_add + + @freeze_time(FROZEN_TIME) @mock.patch("requests_ntlm.HttpNtlmAuth") def test_powerbi_ingest(mock_msal, pytestconfig, tmp_path, mock_time, requests_mock): @@ -174,34 +169,54 @@ def test_powerbi_ingest(mock_msal, pytestconfig, tmp_path, mock_time, requests_m register_mock_api(request_mock=requests_mock) pipeline = Pipeline.create( - { - "run_id": "powerbi-report-server-test", - "source": { - "type": "powerbi-report-server", - "config": { - **default_source_config(), - }, - }, - "sink": { - "type": "file", - "config": { - "filename": f"{tmp_path}/powerbi_report_server_mces.json", - }, - }, - } + get_default_recipe(output_path=f"{tmp_path}/powerbi_report_server_mces.json") ) - pipeline.ctx.graph = mock.MagicMock() - pipeline.ctx.graph.get_ownership = mock.MagicMock() - pipeline.ctx.graph.get_ownership.side_effect = mock_existing_users - pipeline.ctx.graph.get_aspect_v2 = mock.MagicMock() - pipeline.ctx.graph.get_aspect_v2.side_effect = mock_user_to_add + + add_mock_method_in_pipeline(pipeline=pipeline) + + pipeline.run() + pipeline.raise_from_status() + + golden_file = "golden_test_ingest.json" + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / "powerbi_report_server_mces.json", + golden_path=f"{test_resources_dir}/{golden_file}", + ) + + +@freeze_time(FROZEN_TIME) +@mock.patch("requests_ntlm.HttpNtlmAuth") +def test_powerbi_ingest_with_failure( + mock_msal, pytestconfig, tmp_path, mock_time, requests_mock +): + test_resources_dir = ( + pytestconfig.rootpath / "tests/integration/powerbi_report_server" + ) + + register_mock_api( + request_mock=requests_mock, + override_mock_data={ + "https://host_port/Reports/api/v2.0/LinkedReports": { + "method": "GET", + "status_code": 404, + "json": {"error": "Request Failed"}, + } + }, + ) + + pipeline = Pipeline.create( + get_default_recipe(output_path=f"{tmp_path}/powerbi_report_server_mces.json") + ) + + add_mock_method_in_pipeline(pipeline=pipeline) pipeline.run() pipeline.raise_from_status() - mce_out_file = "golden_test_ingest.json" + golden_file = "golden_test_fail_api_ingest.json" mce_helpers.check_golden_file( pytestconfig, output_path=tmp_path / "powerbi_report_server_mces.json", - golden_path=f"{test_resources_dir}/{mce_out_file}", + golden_path=f"{test_resources_dir}/{golden_file}", ) From 056d3619f0923757c1e4afe9f0e5a56de1a64dde Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Fri, 21 Jul 2023 14:14:06 -0700 Subject: [PATCH 172/222] feat(sdk): easily generate container urns (#8198) Co-authored-by: Aseem Bansal --- docs/how/updating-datahub.md | 2 ++ .../src/datahub/emitter/mce_builder.py | 21 ++++++++++++-- .../src/datahub/emitter/mcp_builder.py | 29 ++++++++++++------- .../src/datahub/ingestion/graph/client.py | 13 +++++---- .../ingestion/source/bigquery_v2/bigquery.py | 6 ++-- .../data_lake_common/data_lake_utils.py | 4 +-- .../ingestion/source/powerbi/powerbi.py | 5 ++-- .../powerbi/rest_api_wrapper/data_classes.py | 10 +++---- .../datahub/ingestion/source/sql/athena.py | 6 ++-- .../datahub/ingestion/source/sql/sql_utils.py | 14 ++++----- .../source/sql/two_tier_sql_source.py | 8 ++--- .../src/datahub/ingestion/source/tableau.py | 8 ++--- .../datahub/ingestion/source/unity/source.py | 4 +-- 13 files changed, 78 insertions(+), 52 deletions(-) diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 03b3d763ed247..b705c973cdbb5 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -20,6 +20,8 @@ individually enable / disable desired field metrics. ### Deprecations +- #8198: In the Python SDK, the `PlatformKey` class has been renamed to `ContainerKey`. + ### Other notable Changes ## 0.10.4 diff --git a/metadata-ingestion/src/datahub/emitter/mce_builder.py b/metadata-ingestion/src/datahub/emitter/mce_builder.py index 9c44949741297..47727d5784a19 100644 --- a/metadata-ingestion/src/datahub/emitter/mce_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mce_builder.py @@ -6,7 +6,17 @@ import time from enum import Enum from hashlib import md5 -from typing import Any, List, Optional, Type, TypeVar, Union, cast, get_type_hints +from typing import ( + TYPE_CHECKING, + Any, + List, + Optional, + Type, + TypeVar, + Union, + cast, + get_type_hints, +) import typing_inspect @@ -50,6 +60,9 @@ os.getenv("DATAHUB_DATASET_URN_TO_LOWER", "false") == "true" ) +if TYPE_CHECKING: + from datahub.emitter.mcp_builder import DatahubKey + # TODO: Delete this once lower-casing is the standard. def set_dataset_urn_to_lower(value: bool) -> None: @@ -132,7 +145,11 @@ def dataset_key_to_urn(key: DatasetKeyClass) -> str: ) -def make_container_urn(guid: str) -> str: +def make_container_urn(guid: Union[str, "DatahubKey"]) -> str: + from datahub.emitter.mcp_builder import DatahubKey + + if isinstance(guid, DatahubKey): + guid = guid.guid() return f"urn:li:container:{guid}" diff --git a/metadata-ingestion/src/datahub/emitter/mcp_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_builder.py index 9051f2e82fa1f..40df214f49433 100644 --- a/metadata-ingestion/src/datahub/emitter/mcp_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mcp_builder.py @@ -54,7 +54,9 @@ def guid(self) -> str: return _stable_guid_from_dict(bag) -class PlatformKey(DatahubKey): +class ContainerKey(DatahubKey): + """Base class for container guid keys. Most users should use one of the subclasses instead.""" + platform: str instance: Optional[str] = None @@ -81,8 +83,15 @@ def guid_dict(self) -> Dict[str, str]: def property_dict(self) -> Dict[str, str]: return self.dict(by_alias=True, exclude_none=True) + def as_urn(self) -> str: + return make_container_urn(guid=self.guid()) + + +# DEPRECATION: Keeping the `PlatformKey` name around for backwards compatibility. +PlatformKey = ContainerKey + -class DatabaseKey(PlatformKey): +class DatabaseKey(ContainerKey): database: str @@ -90,11 +99,11 @@ class SchemaKey(DatabaseKey): db_schema: str = Field(alias="schema") -class ProjectIdKey(PlatformKey): +class ProjectIdKey(ContainerKey): project_id: str -class MetastoreKey(PlatformKey): +class MetastoreKey(ContainerKey): metastore: str @@ -110,11 +119,11 @@ class BigQueryDatasetKey(ProjectIdKey): dataset_id: str -class FolderKey(PlatformKey): +class FolderKey(ContainerKey): folder_abs_path: str -class BucketKey(PlatformKey): +class BucketKey(ContainerKey): bucket_name: str @@ -127,7 +136,7 @@ def default(self, obj: Any) -> Any: return json.JSONEncoder.default(self, obj) -KeyType = TypeVar("KeyType", bound=PlatformKey) +KeyType = TypeVar("KeyType", bound=ContainerKey) def add_domain_to_entity_wu( @@ -188,7 +197,7 @@ def gen_containers( container_key: KeyType, name: str, sub_types: List[str], - parent_container_key: Optional[PlatformKey] = None, + parent_container_key: Optional[ContainerKey] = None, extra_properties: Optional[Dict[str, str]] = None, domain_urn: Optional[str] = None, description: Optional[str] = None, @@ -199,9 +208,7 @@ def gen_containers( created: Optional[int] = None, last_modified: Optional[int] = None, ) -> Iterable[MetadataWorkUnit]: - container_urn = make_container_urn( - guid=container_key.guid(), - ) + container_urn = container_key.as_urn() yield MetadataChangeProposalWrapper( entityUrn=f"{container_urn}", # entityKeyAspect=ContainerKeyClass(guid=parent_container_key.guid()), diff --git a/metadata-ingestion/src/datahub/ingestion/graph/client.py b/metadata-ingestion/src/datahub/ingestion/graph/client.py index 2f817ee69a637..cac53c350f2ea 100644 --- a/metadata-ingestion/src/datahub/ingestion/graph/client.py +++ b/metadata-ingestion/src/datahub/ingestion/graph/client.py @@ -88,6 +88,12 @@ class RemovedStatusFilter(enum.Enum): """Search only soft-deleted entities.""" +@dataclass +class RelatedEntity: + urn: str + relationship_type: str + + def _graphql_entity_type(entity_type: str) -> str: """Convert the entity types into GraphQL "EntityType" enum values.""" @@ -769,11 +775,6 @@ class RelationshipDirection(str, enum.Enum): INCOMING = "INCOMING" OUTGOING = "OUTGOING" - @dataclass - class RelatedEntity: - urn: str - relationship_type: str - def get_related_entities( self, entity_urn: str, @@ -794,7 +795,7 @@ def get_related_entities( }, ) for related_entity in response.get("entities", []): - yield DataHubGraph.RelatedEntity( + yield RelatedEntity( urn=related_entity["urn"], relationship_type=related_entity["relationshipType"], ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py index 919c803222066..ccda00ba293ef 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py @@ -20,7 +20,7 @@ set_dataset_urn_to_lower, ) from datahub.emitter.mcp import MetadataChangeProposalWrapper -from datahub.emitter.mcp_builder import BigQueryDatasetKey, PlatformKey, ProjectIdKey +from datahub.emitter.mcp_builder import BigQueryDatasetKey, ContainerKey, ProjectIdKey from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import ( SupportStatus, @@ -434,7 +434,7 @@ def get_dataplatform_instance_aspect( entityUrn=dataset_urn, aspect=aspect ).as_workunit() - def gen_dataset_key(self, db_name: str, schema: str) -> PlatformKey: + def gen_dataset_key(self, db_name: str, schema: str) -> ContainerKey: return BigQueryDatasetKey( project_id=db_name, dataset_id=schema, @@ -443,7 +443,7 @@ def gen_dataset_key(self, db_name: str, schema: str) -> PlatformKey: backcompat_env_as_instance=True, ) - def gen_project_id_key(self, database: str) -> PlatformKey: + def gen_project_id_key(self, database: str) -> ContainerKey: return ProjectIdKey( project_id=database, platform=self.platform, diff --git a/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/data_lake_utils.py b/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/data_lake_utils.py index 0a65537772390..b04718a9eabba 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/data_lake_utils.py +++ b/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/data_lake_utils.py @@ -3,9 +3,9 @@ from datahub.emitter.mcp_builder import ( BucketKey, + ContainerKey, FolderKey, KeyType, - PlatformKey, add_dataset_to_container, gen_containers, ) @@ -45,7 +45,7 @@ def create_emit_containers( container_key: KeyType, name: str, sub_types: List[str], - parent_container_key: Optional[PlatformKey] = None, + parent_container_key: Optional[ContainerKey] = None, domain_urn: Optional[str] = None, ) -> Iterable[MetadataWorkUnit]: if container_key.guid() not in self.processed_containers: diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py index 33596091e420d..919cb83e4d832 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py @@ -9,7 +9,7 @@ import datahub.emitter.mce_builder as builder import datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes as powerbi_data_classes from datahub.emitter.mcp import MetadataChangeProposalWrapper -from datahub.emitter.mcp_builder import PlatformKey, gen_containers +from datahub.emitter.mcp_builder import ContainerKey, gen_containers from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import ( SourceCapability, @@ -104,7 +104,7 @@ def __init__( self.__reporter = reporter self.__dataplatform_instance_resolver = dataplatform_instance_resolver self.processed_datasets: Set[powerbi_data_classes.PowerBIDataset] = set() - self.workspace_key: PlatformKey + self.workspace_key: ContainerKey @staticmethod def urn_to_lowercase(value: str, flag: bool) -> str: @@ -256,7 +256,6 @@ def to_datahub_schema( self, table: powerbi_data_classes.Table, ) -> SchemaMetadataClass: - fields = [] table_fields = ( [self.to_datahub_schema_field(column) for column in table.columns] diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py index 28a5fac8b127b..2d2d9f527788f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py @@ -2,7 +2,7 @@ from enum import Enum from typing import Any, Dict, List, Optional, Union -from datahub.emitter.mcp_builder import PlatformKey +from datahub.emitter.mcp_builder import ContainerKey from datahub.metadata.schema_classes import ( BooleanTypeClass, DateTypeClass, @@ -28,11 +28,11 @@ } -class WorkspaceKey(PlatformKey): +class WorkspaceKey(ContainerKey): workspace: str -class DatasetKey(PlatformKey): +class DatasetKey(ContainerKey): dataset: str @@ -57,7 +57,7 @@ def get_workspace_key( platform_name: str, platform_instance: Optional[str] = None, workspace_id_as_urn_part: Optional[bool] = False, - ) -> PlatformKey: + ) -> ContainerKey: return WorkspaceKey( workspace=self.get_urn_part(workspace_id_as_urn_part), platform=platform_name, @@ -150,7 +150,7 @@ def __eq__(self, instance): def __hash__(self): return hash(self.__members()) - def get_dataset_key(self, platform_name: str) -> PlatformKey: + def get_dataset_key(self, platform_name: str) -> ContainerKey: return DatasetKey( dataset=self.id, platform=platform_name, diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py index 46f9fd240db04..8b2eed36ac6b3 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py @@ -9,7 +9,7 @@ from sqlalchemy.engine.reflection import Inspector from datahub.configuration.validate_field_rename import pydantic_renamed_field -from datahub.emitter.mcp_builder import PlatformKey +from datahub.emitter.mcp_builder import ContainerKey from datahub.ingestion.api.decorators import ( SourceCapability, SupportStatus, @@ -211,7 +211,7 @@ def gen_schema_containers( extra_properties=extra_properties, ) - def get_database_container_key(self, db_name: str, schema: str) -> PlatformKey: + def get_database_container_key(self, db_name: str, schema: str) -> ContainerKey: # Because our overridden get_allowed_schemas method returns db_name as the schema name, # the db_name and schema here will be the same. Hence, we just ignore the schema parameter. # Based on community feedback, db_name only available if it is explicitly specified in the connection string. @@ -232,7 +232,7 @@ def add_table_to_schema_container( dataset_urn: str, db_name: str, schema: str, - schema_container_key: Optional[PlatformKey] = None, + schema_container_key: Optional[ContainerKey] = None, ) -> Iterable[MetadataWorkUnit]: yield from add_table_to_schema_container( dataset_urn=dataset_urn, diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_utils.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_utils.py index 7554dd5af3103..a5f5034d175c6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_utils.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_utils.py @@ -8,8 +8,8 @@ ) from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.mcp_builder import ( + ContainerKey, DatabaseKey, - PlatformKey, SchemaKey, add_dataset_to_container, add_domain_to_entity_wu, @@ -28,7 +28,7 @@ def gen_schema_key( platform: str, platform_instance: Optional[str], env: Optional[str], -) -> PlatformKey: +) -> ContainerKey: return SchemaKey( database=db_name, schema=schema, @@ -41,7 +41,7 @@ def gen_schema_key( def gen_database_key( database: str, platform: str, platform_instance: Optional[str], env: Optional[str] -) -> PlatformKey: +) -> ContainerKey: return DatabaseKey( database=database, platform=platform, @@ -55,8 +55,8 @@ def gen_schema_container( schema: str, database: str, sub_types: List[str], - database_container_key: PlatformKey, - schema_container_key: PlatformKey, + database_container_key: ContainerKey, + schema_container_key: ContainerKey, domain_registry: Optional[DomainRegistry] = None, domain_config: Optional[Dict[str, AllowDenyPattern]] = None, name: Optional[str] = None, @@ -113,7 +113,7 @@ def gen_domain_urn( def gen_database_container( database: str, - database_container_key: PlatformKey, + database_container_key: ContainerKey, sub_types: List[str], domain_config: Optional[Dict[str, AllowDenyPattern]] = None, domain_registry: Optional[DomainRegistry] = None, @@ -152,7 +152,7 @@ def gen_database_container( def add_table_to_schema_container( dataset_urn: str, - parent_container_key: PlatformKey, + parent_container_key: ContainerKey, ) -> Iterable[MetadataWorkUnit]: yield from add_dataset_to_container( container_key=parent_container_key, diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py b/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py index f105829d874de..d9062cef06eae 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py @@ -7,7 +7,7 @@ from datahub.configuration.common import AllowDenyPattern from datahub.configuration.validate_field_rename import pydantic_renamed_field -from datahub.emitter.mcp_builder import PlatformKey +from datahub.emitter.mcp_builder import ContainerKey from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.sql.sql_common import SQLAlchemySource, logger from datahub.ingestion.source.sql.sql_config import ( @@ -56,7 +56,7 @@ def __init__(self, config, ctx, platform): super().__init__(config, ctx, platform) self.config: TwoTierSQLAlchemyConfig = config - def get_database_container_key(self, db_name: str, schema: str) -> PlatformKey: + def get_database_container_key(self, db_name: str, schema: str) -> ContainerKey: # Because our overridden get_allowed_schemas method returns db_name as the schema name, # the db_name and schema here will be the same. Hence, we just ignore the schema parameter. assert db_name == schema @@ -72,7 +72,7 @@ def add_table_to_schema_container( dataset_urn: str, db_name: str, schema: str, - schema_container_key: Optional[PlatformKey] = None, + schema_container_key: Optional[ContainerKey] = None, ) -> Iterable[MetadataWorkUnit]: yield from add_table_to_schema_container( dataset_urn=dataset_urn, @@ -86,7 +86,7 @@ def get_allowed_schemas( # dbName itself as an allowed schema yield db_name - def gen_schema_key(self, db_name: str, schema: str) -> PlatformKey: + def gen_schema_key(self, db_name: str, schema: str) -> ContainerKey: # Sanity check that we don't try to generate schema containers for 2 tier databases. raise NotImplementedError diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau.py index 5ad39425c3f73..67bd1af6c2d7f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau.py @@ -33,7 +33,7 @@ ) from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.mcp_builder import ( - PlatformKey, + ContainerKey, add_entity_to_container, gen_containers, ) @@ -358,11 +358,11 @@ def projects_backward_compatibility(cls, values: Dict) -> Dict: return values -class WorkbookKey(PlatformKey): +class WorkbookKey(ContainerKey): workbook_id: str -class ProjectKey(PlatformKey): +class ProjectKey(ContainerKey): project_id: str @@ -1682,7 +1682,7 @@ def emit_datasource( ) def _get_datasource_container_key(self, datasource, workbook, is_embedded_ds): - container_key: Optional[PlatformKey] = None + container_key: Optional[ContainerKey] = None if is_embedded_ds: # It is embedded then parent is container is workbook if workbook is not None: container_key = self.gen_workbook_key(workbook) diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py index 9d82a9e247a00..ec7d00c7bcc63 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py @@ -15,8 +15,8 @@ from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.mcp_builder import ( CatalogKey, + ContainerKey, MetastoreKey, - PlatformKey, UnitySchemaKey, add_dataset_to_container, gen_containers, @@ -432,7 +432,7 @@ def gen_catalog_containers(self, catalog: Catalog) -> Iterable[MetadataWorkUnit] external_url=f"{self.external_url_base}/{catalog.name}", ) - def gen_schema_key(self, schema: Schema) -> PlatformKey: + def gen_schema_key(self, schema: Schema) -> ContainerKey: return UnitySchemaKey( unity_schema=schema.name, platform=self.platform, From a97ac524816943b9bca0c5294434ea8f5e94b3db Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Sun, 23 Jul 2023 04:40:22 -0700 Subject: [PATCH 173/222] fix(presto-on-hive): update presto-on-hive URN in data_platforms.json (#8484) --- .../war/src/main/resources/boot/data_platforms.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-service/war/src/main/resources/boot/data_platforms.json b/metadata-service/war/src/main/resources/boot/data_platforms.json index fc285353f5005..2abe81d93236c 100644 --- a/metadata-service/war/src/main/resources/boot/data_platforms.json +++ b/metadata-service/war/src/main/resources/boot/data_platforms.json @@ -427,7 +427,7 @@ } }, { - "urn": "urn:li:dataPlatform:presto_on_hive", + "urn": "urn:li:dataPlatform:presto-on-hive", "aspect": { "datasetNameDelimiter": ".", "name": "presto-on-hive", From 8fb5912978d35200367a68b7bcd69a69f93a540e Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Sun, 23 Jul 2023 17:10:54 +0530 Subject: [PATCH 174/222] fix(mysql): getting table name correctly (#8476) Co-authored-by: Ellie O'Neil --- .../ingestion/source/ge_data_profiler.py | 16 +- .../mysql_table_row_count_estimate_only.json | 154 ++++++++++++++++++ 2 files changed, 164 insertions(+), 6 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py index 4ea721f6fd0cc..6faa29f264d36 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py @@ -360,18 +360,22 @@ def _get_column_cardinality( @_run_with_query_combiner def _get_dataset_rows(self, dataset_profile: DatasetProfileClass) -> None: if self.config.profile_table_row_count_estimate_only: - schema_name = self.dataset_name.split(".")[1] - table_name = self.dataset_name.split(".")[2] - logger.debug( - f"Getting estimated rowcounts for table:{self.dataset_name}, schema:{schema_name}, table:{table_name}" - ) - dialect_name = self.dataset.engine.dialect.name.lower() if dialect_name == "postgresql": + schema_name = self.dataset_name.split(".")[1] + table_name = self.dataset_name.split(".")[2] + logger.debug( + f"Getting estimated rowcounts for table:{self.dataset_name}, schema:{schema_name}, table:{table_name}" + ) get_estimate_script = sa.text( f"SELECT c.reltuples AS estimate FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relname = '{table_name}' AND n.nspname = '{schema_name}'" ) elif dialect_name == "mysql": + schema_name = self.dataset_name.split(".")[0] + table_name = self.dataset_name.split(".")[1] + logger.debug( + f"Getting estimated rowcounts for table:{self.dataset_name}, schema:{schema_name}, table:{table_name}" + ) get_estimate_script = sa.text( f"SELECT table_rows AS estimate FROM information_schema.tables WHERE table_schema = '{schema_name}' AND table_name = '{table_name}'" ) diff --git a/metadata-ingestion/tests/integration/mysql/mysql_table_row_count_estimate_only.json b/metadata-ingestion/tests/integration/mysql/mysql_table_row_count_estimate_only.json index e668525b930af..7597013bd873a 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_table_row_count_estimate_only.json +++ b/metadata-ingestion/tests/integration/mysql/mysql_table_row_count_estimate_only.json @@ -400,5 +400,159 @@ "lastObserved": 1586847600000, "runId": "mysql-2020_04_14-07_00_00" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1586847600000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 0, + "columnCount": 6, + "fieldProfiles": [ + { + "fieldPath": "id", + "uniqueCount": 5, + "uniqueProportion": 1, + "nullCount": 0, + "min": "1", + "max": "5", + "mean": "3.0", + "median": "3", + "stdev": "1.5811388300841898", + "sampleValues": [ + "1", + "2", + "3", + "4", + "5" + ] + }, + { + "fieldPath": "company", + "uniqueCount": 5, + "uniqueProportion": 1, + "nullCount": 0, + "sampleValues": [ + "Company A", + "Company B", + "Company C", + "Company D", + "Company E" + ] + }, + { + "fieldPath": "last_name", + "uniqueCount": 5, + "uniqueProportion": 1, + "nullCount": 0, + "sampleValues": [ + "Axen", + "Bedecs", + "Donnell", + "Gratacos Solsona", + "Lee" + ] + }, + { + "fieldPath": "first_name", + "uniqueCount": 5, + "uniqueProportion": 1, + "nullCount": 0, + "sampleValues": [ + "Anna", + "Antonio", + "Christina", + "Martin", + "Thomas" + ] + }, + { + "fieldPath": "email_address", + "uniqueCount": 0, + "nullCount": 0, + "sampleValues": [] + }, + { + "fieldPath": "priority", + "uniqueCount": 3, + "uniqueProportion": 0.75, + "nullCount": 0, + "min": "3.8", + "max": "4.9", + "mean": "4.175000011920929", + "median": "4.0", + "stdev": "0.49244294899530355", + "sampleValues": [ + "4.0", + "4.9", + "4.0", + "3.8" + ] + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1586847600000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 0, + "columnCount": 3, + "fieldProfiles": [ + { + "fieldPath": "id", + "uniqueCount": 0, + "nullCount": 0, + "min": "None", + "max": "None", + "mean": "None", + "median": "None", + "stdev": "0.0", + "sampleValues": [] + }, + { + "fieldPath": "description", + "uniqueCount": 0, + "nullCount": 0, + "sampleValues": [] + }, + { + "fieldPath": "customer_id", + "uniqueCount": 0, + "nullCount": 0, + "min": "None", + "max": "None", + "mean": "None", + "median": "None", + "stdev": "0.0", + "sampleValues": [] + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-2020_04_14-07_00_00" + } } ] \ No newline at end of file From c0dbea83633000dbe0171b7f34a894d8dff9cee6 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Sun, 23 Jul 2023 17:12:31 +0530 Subject: [PATCH 175/222] feat(ingest/elastic): reduce number of calls made (#8477) --- .../ingestion/source/elastic_search.py | 75 ++++++++++--------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py b/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py index ab6aa18dac4d2..7fac85627ed8b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py +++ b/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py @@ -343,7 +343,7 @@ def __init__(self, config: ElasticsearchSourceConfig, ctx: PipelineContext): self.report = ElasticsearchSourceReport() self.data_stream_partition_count: Dict[str, int] = defaultdict(int) self.platform: str = "elasticsearch" - self.profiling_info: Dict[str, DatasetProfileClass] = {} + self.cat_response: Optional[List[Dict[str, Any]]] = None @classmethod def create( @@ -357,7 +357,6 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: indices = self.client.indices.get_alias() - for index in indices: self.report.report_index_scanned(index) @@ -366,12 +365,6 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: yield mcp.as_workunit() else: self.report.report_dropped(index) - for urn, profiling_info in self.profiling_info.items(): - yield MetadataChangeProposalWrapper( - entityUrn=urn, - aspect=profiling_info, - ).as_workunit() - self.profiling_info = {} for mcp in self._get_data_stream_index_count_mcps(): yield mcp.as_workunit() @@ -523,36 +516,44 @@ def _extract_mcps( ) if self.source_config.profiling.enabled: - cat_response = self.client.cat.indices( - index=index, params={"format": "json", "bytes": "b"} + if self.cat_response is None: + self.cat_response = self.client.cat.indices( + params={ + "format": "json", + "bytes": "b", + "h": "index,docs.count,store.size", + } + ) + if self.cat_response is None: + return + for item in self.cat_response: + item["index"] = collapse_name( + name=item["index"], + collapse_urns=self.source_config.collapse_urns, + ) + + profile_info_current = list( + filter(lambda x: x["index"] == collapsed_index_name, self.cat_response) ) - if len(cat_response) == 1: - index_res = cat_response[0] - docs_count = int(index_res["docs.count"]) - size = int(index_res["store.size"]) - if len(self.source_config.collapse_urns.urns_suffix_regex) > 0: - if dataset_urn not in self.profiling_info: - self.profiling_info[dataset_urn] = DatasetProfileClass( - timestampMillis=int(time.time() * 1000), - rowCount=docs_count, - columnCount=len(schema_fields), - sizeInBytes=size, - ) - else: - existing_profile = self.profiling_info[dataset_urn] - if existing_profile.rowCount is not None: - docs_count = docs_count + existing_profile.rowCount - if existing_profile.sizeInBytes is not None: - size = size + existing_profile.sizeInBytes - self.profiling_info[dataset_urn] = DatasetProfileClass( - timestampMillis=int(time.time() * 1000), - rowCount=docs_count, - columnCount=len(schema_fields), - sizeInBytes=size, - ) - else: - logger.warning( - "Unexpected response from cat response with multiple rows" + if len(profile_info_current) > 0: + self.cat_response = list( + filter( + lambda x: x["index"] != collapsed_index_name, self.cat_response + ) + ) + row_count = 0 + size_in_bytes = 0 + for profile_info in profile_info_current: + row_count += int(profile_info["docs.count"]) + size_in_bytes += int(profile_info["store.size"]) + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=DatasetProfileClass( + timestampMillis=int(time.time() * 1000), + rowCount=row_count, + columnCount=len(schema_fields), + sizeInBytes=size_in_bytes, + ), ) def get_report(self): From 27392f9f81e35c95e24588ad5c26235ec0f4b2d3 Mon Sep 17 00:00:00 2001 From: Indy Prentice Date: Mon, 24 Jul 2023 15:39:44 -0300 Subject: [PATCH 176/222] refactor(search): Support searching multiple entities in search() as in scroll() (#8461) Co-authored-by: Indy Prentice --- .../metadata/client/JavaEntityClient.java | 4 +- .../metadata/search/SearchService.java | 6 +- .../AllEntitiesSearchAggregator.java | 2 +- .../client/CachingEntitySearchService.java | 14 +-- .../elasticsearch/ElasticSearchService.java | 10 +- .../elasticsearch/query/ESSearchDAO.java | 16 +-- .../com/linkedin/metadata/ESTestUtils.java | 15 ++- .../ElasticSearchServiceTest.java | 19 ++-- .../fixtures/SampleDataFixtureTests.java | 98 +++++++++++-------- .../SearchLineageDataFixtureTests.java | 6 +- .../hook/siblings/SiblingAssociationHook.java | 2 +- .../siblings/SiblingAssociationHookTest.java | 2 +- .../boot/steps/RestoreGlossaryIndices.java | 4 +- .../steps/RestoreGlossaryIndicesTest.java | 9 +- .../resources/entity/EntityResource.java | 2 +- .../metadata/search/EntitySearchService.java | 8 +- .../src/main/java/mock/MockEntitySpec.java | 4 +- 17 files changed, 125 insertions(+), 96 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java b/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java index 911ab993e5789..e816700e583c7 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java @@ -277,7 +277,7 @@ public SearchResult search(@Nonnull String entity, @Nonnull String input, @Nullable SearchFlags searchFlags) throws RemoteInvocationException { - return ValidationUtils.validateSearchResult(_entitySearchService.search(entity, input, newFilter(requestFilters), + return ValidationUtils.validateSearchResult(_entitySearchService.search(List.of(entity), input, newFilter(requestFilters), null, start, count, searchFlags), _entityService); } @@ -329,7 +329,7 @@ public SearchResult search( @Nullable SearchFlags searchFlags) throws RemoteInvocationException { return ValidationUtils.validateSearchResult( - _entitySearchService.search(entity, input, filter, sortCriterion, start, count, searchFlags), _entityService); + _entitySearchService.search(List.of(entity), input, filter, sortCriterion, start, count, searchFlags), _entityService); } @Nonnull diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java index f3c0d14bddac3..a14ea8d36b0c1 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java @@ -44,7 +44,7 @@ public Map docCountPerEntity(@Nonnull List entityNames) { * Gets a list of documents that match given search request. The results are aggregated and filters are applied to the * search hits and not the aggregation results. * - * @param entityName name of the entity + * @param entityNames names of the entity * @param input the search input text * @param postFilters the request map with fields and values as filters to be applied to search hits * @param sortCriterion {@link SortCriterion} to be applied to search results @@ -54,10 +54,10 @@ public Map docCountPerEntity(@Nonnull List entityNames) { * @return a {@link SearchResult} that contains a list of matched documents and related search result metadata */ @Nonnull - public SearchResult search(@Nonnull String entityName, @Nonnull String input, @Nullable Filter postFilters, + public SearchResult search(@Nonnull List entityNames, @Nonnull String input, @Nullable Filter postFilters, @Nullable SortCriterion sortCriterion, int from, int size, @Nullable SearchFlags searchFlags) { SearchResult result = - _cachingEntitySearchService.search(entityName, input, postFilters, sortCriterion, from, size, searchFlags, null); + _cachingEntitySearchService.search(entityNames, input, postFilters, sortCriterion, from, size, searchFlags, null); try { return result.copy().setEntities(new SearchEntityArray(_searchRanker.rank(result.getEntities()))); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/aggregator/AllEntitiesSearchAggregator.java b/metadata-io/src/main/java/com/linkedin/metadata/search/aggregator/AllEntitiesSearchAggregator.java index ee93edaf2480c..1af94141366e1 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/aggregator/AllEntitiesSearchAggregator.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/aggregator/AllEntitiesSearchAggregator.java @@ -171,7 +171,7 @@ private Map getSearchResultsForEachEntity(@Nonnull List new Pair<>(entity, - _cachingEntitySearchService.search(entity, input, postFilters, sortCriterion, queryFrom, querySize, searchFlags, facets))) + _cachingEntitySearchService.search(List.of(entity), input, postFilters, sortCriterion, queryFrom, querySize, searchFlags, facets))) .stream() .collect(Collectors.toMap(Pair::getKey, Pair::getValue)); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java index 56f6fed3ad9d2..f698e28c0be6d 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java @@ -53,7 +53,7 @@ public class CachingEntitySearchService { * @return a {@link SearchResult} containing the requested batch of search results */ public SearchResult search( - @Nonnull String entityName, + @Nonnull List entityNames, @Nonnull String query, @Nullable Filter filters, @Nullable SortCriterion sortCriterion, @@ -61,7 +61,7 @@ public SearchResult search( int size, @Nullable SearchFlags flags, @Nullable List facets) { - return getCachedSearchResults(entityName, query, filters, sortCriterion, from, size, flags, facets); + return getCachedSearchResults(entityNames, query, filters, sortCriterion, from, size, flags, facets); } /** @@ -141,7 +141,7 @@ public ScrollResult scroll( * This lets us have batches that return a variable number of results (we have no idea which batch the "from" "size" page corresponds to) */ public SearchResult getCachedSearchResults( - @Nonnull String entityName, + @Nonnull List entityNames, @Nonnull String query, @Nullable Filter filters, @Nullable SortCriterion sortCriterion, @@ -152,9 +152,9 @@ public SearchResult getCachedSearchResults( return new CacheableSearcher<>( cacheManager.getCache(ENTITY_SEARCH_SERVICE_SEARCH_CACHE_NAME), batchSize, - querySize -> getRawSearchResults(entityName, query, filters, sortCriterion, querySize.getFrom(), + querySize -> getRawSearchResults(entityNames, query, filters, sortCriterion, querySize.getFrom(), querySize.getSize(), flags, facets), - querySize -> Sextet.with(entityName, query, filters != null ? toJsonString(filters) : null, + querySize -> Sextet.with(entityNames, query, filters != null ? toJsonString(filters) : null, sortCriterion != null ? toJsonString(sortCriterion) : null, facets, querySize), flags, enableCache).getSearchResults(from, size); } @@ -272,7 +272,7 @@ public ScrollResult getCachedScrollResults( * Executes the expensive search query using the {@link EntitySearchService} */ private SearchResult getRawSearchResults( - final String entityName, + final List entityNames, final String input, final Filter filters, final SortCriterion sortCriterion, @@ -280,7 +280,7 @@ private SearchResult getRawSearchResults( final int count, @Nullable final SearchFlags searchFlags, @Nullable final List facets) { - return entitySearchService.search(entityName, input, filters, sortCriterion, start, count, searchFlags, facets); + return entitySearchService.search(entityNames, input, filters, sortCriterion, start, count, searchFlags, facets); } /** diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java index 0680e6d74edf3..ce7b44c715d6b 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java @@ -106,18 +106,18 @@ public void appendRunId(@Nonnull String entityName, @Nonnull Urn urn, @Nullable @Nonnull @Override - public SearchResult search(@Nonnull String entityName, @Nonnull String input, @Nullable Filter postFilters, + public SearchResult search(@Nonnull List entityNames, @Nonnull String input, @Nullable Filter postFilters, @Nullable SortCriterion sortCriterion, int from, int size, @Nullable SearchFlags searchFlags) { - return search(entityName, input, postFilters, sortCriterion, from, size, searchFlags, null); + return search(entityNames, input, postFilters, sortCriterion, from, size, searchFlags, null); } @Nonnull - public SearchResult search(@Nonnull String entityName, @Nonnull String input, @Nullable Filter postFilters, + public SearchResult search(@Nonnull List entityNames, @Nonnull String input, @Nullable Filter postFilters, @Nullable SortCriterion sortCriterion, int from, int size, @Nullable SearchFlags searchFlags, @Nullable List facets) { log.debug(String.format( "Searching FullText Search documents entityName: %s, input: %s, postFilters: %s, sortCriterion: %s, from: %s, size: %s", - entityName, input, postFilters, sortCriterion, from, size)); - return esSearchDAO.search(entityName, input, postFilters, sortCriterion, from, size, searchFlags, facets); + entityNames, input, postFilters, sortCriterion, from, size)); + return esSearchDAO.search(entityNames, input, postFilters, sortCriterion, from, size, searchFlags, facets); } @Nonnull diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java index 51e08763cd7c8..57e8967c83985 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java @@ -79,7 +79,7 @@ public long docCount(@Nonnull String entityName) { @Nonnull @WithSpan - private SearchResult executeAndExtract(@Nonnull EntitySpec entitySpec, @Nonnull SearchRequest searchRequest, + private SearchResult executeAndExtract(@Nonnull List entitySpec, @Nonnull SearchRequest searchRequest, @Nullable Filter filter, int from, int size) { long id = System.currentTimeMillis(); try (Timer.Context ignored = MetricUtils.timer(this.getClass(), "executeAndExtract_search").time()) { @@ -181,20 +181,22 @@ private ScrollResult executeAndExtract(@Nonnull List entitySpecs, @N * @return a {@link SearchResult} that contains a list of matched documents and related search result metadata */ @Nonnull - public SearchResult search(@Nonnull String entityName, @Nonnull String input, @Nullable Filter postFilters, + public SearchResult search(@Nonnull List entityNames, @Nonnull String input, @Nullable Filter postFilters, @Nullable SortCriterion sortCriterion, int from, int size, @Nullable SearchFlags searchFlags, @Nullable List facets) { final String finalInput = input.isEmpty() ? "*" : input; Timer.Context searchRequestTimer = MetricUtils.timer(this.getClass(), "searchRequest").time(); - EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); + List entitySpecs = entityNames.stream().map(entityRegistry::getEntitySpec).collect(Collectors.toList()); Filter transformedFilters = transformFilterForEntities(postFilters, indexConvention); // Step 1: construct the query final SearchRequest searchRequest = SearchRequestHandler - .getBuilder(entitySpec, searchConfiguration, customSearchConfiguration) + .getBuilder(entitySpecs, searchConfiguration, customSearchConfiguration) .getSearchRequest(finalInput, transformedFilters, sortCriterion, from, size, searchFlags, facets); - searchRequest.indices(indexConvention.getIndexName(entitySpec)); + searchRequest.indices(entityNames.stream() + .map(indexConvention::getEntityIndexName) + .toArray(String[]::new)); searchRequestTimer.stop(); // Step 2: execute the query and extract results, validated against document model as well - return executeAndExtract(entitySpec, searchRequest, transformedFilters, from, size); + return executeAndExtract(entitySpecs, searchRequest, transformedFilters, from, size); } /** @@ -217,7 +219,7 @@ public SearchResult filter(@Nonnull String entityName, @Nullable Filter filters, .getFilterRequest(transformedFilters, sortCriterion, from, size); searchRequest.indices(indexConvention.getIndexName(entitySpec)); - return executeAndExtract(entitySpec, searchRequest, transformedFilters, from, size); + return executeAndExtract(List.of(entitySpec), searchRequest, transformedFilters, from, size); } /** diff --git a/metadata-io/src/test/java/com/linkedin/metadata/ESTestUtils.java b/metadata-io/src/test/java/com/linkedin/metadata/ESTestUtils.java index c1297866edcc4..79496888650e1 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/ESTestUtils.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/ESTestUtils.java @@ -68,15 +68,24 @@ private ESTestUtils() { .collect(Collectors.toList()); } - public static SearchResult search(SearchService searchService, String query) { - return search(searchService, query, null); + public static SearchResult searchAcrossEntities(SearchService searchService, String query) { + return searchAcrossEntities(searchService, query, null); } - public static SearchResult search(SearchService searchService, String query, @Nullable List facets) { + public static SearchResult searchAcrossEntities(SearchService searchService, String query, @Nullable List facets) { return searchService.searchAcrossEntities(SEARCHABLE_ENTITIES, query, null, null, 0, 100, new SearchFlags().setFulltext(true).setSkipCache(true), facets); } + public static SearchResult search(SearchService searchService, String query) { + return search(searchService, SEARCHABLE_ENTITIES, query); + } + + public static SearchResult search(SearchService searchService, List entities, String query) { + return searchService.search(entities, query, null, null, 0, 100, + new SearchFlags().setFulltext(true).setSkipCache(true)); + } + public static ScrollResult scroll(SearchService searchService, String query, int batchSize, @Nullable String scrollId) { return searchService.scrollAcrossEntities(SEARCHABLE_ENTITIES, query, null, null, scrollId, "3m", batchSize, new SearchFlags().setFulltext(true).setSkipCache(true)); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java index 0e8d881b70791..9a6d2dc6fc1fa 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java @@ -23,6 +23,7 @@ import com.linkedin.metadata.search.elasticsearch.update.ESWriteDAO; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import com.linkedin.metadata.utils.elasticsearch.IndexConventionImpl; +import java.util.List; import org.elasticsearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Import; @@ -93,7 +94,7 @@ private ElasticSearchService buildService() { @Test public void testElasticSearchServiceStructuredQuery() throws Exception { - SearchResult searchResult = _elasticSearchService.search(ENTITY_NAME, "test", null, null, 0, 10, new SearchFlags().setFulltext(false)); + SearchResult searchResult = _elasticSearchService.search(List.of(ENTITY_NAME), "test", null, null, 0, 10, new SearchFlags().setFulltext(false)); assertEquals(searchResult.getNumEntities().intValue(), 0); BrowseResult browseResult = _elasticSearchService.browse(ENTITY_NAME, "", null, 0, 10); assertEquals(browseResult.getMetadata().getTotalNumEntities().longValue(), 0); @@ -110,10 +111,10 @@ public void testElasticSearchServiceStructuredQuery() throws Exception { _elasticSearchService.upsertDocument(ENTITY_NAME, document.toString(), urn.toString()); syncAfterWrite(_bulkProcessor); - searchResult = _elasticSearchService.search(ENTITY_NAME, "test", null, null, 0, 10, new SearchFlags().setFulltext(false)); + searchResult = _elasticSearchService.search(List.of(ENTITY_NAME), "test", null, null, 0, 10, new SearchFlags().setFulltext(false)); assertEquals(searchResult.getNumEntities().intValue(), 1); assertEquals(searchResult.getEntities().get(0).getEntity(), urn); - searchResult = _elasticSearchService.search(ENTITY_NAME, "foreignKey:Node", null, null, 0, 10, new SearchFlags().setFulltext(false)); + searchResult = _elasticSearchService.search(List.of(ENTITY_NAME), "foreignKey:Node", null, null, 0, 10, new SearchFlags().setFulltext(false)); assertEquals(searchResult.getNumEntities().intValue(), 1); assertEquals(searchResult.getEntities().get(0).getEntity(), urn); browseResult = _elasticSearchService.browse(ENTITY_NAME, "", null, 0, 10); @@ -135,7 +136,7 @@ public void testElasticSearchServiceStructuredQuery() throws Exception { _elasticSearchService.upsertDocument(ENTITY_NAME, document2.toString(), urn2.toString()); syncAfterWrite(_bulkProcessor); - searchResult = _elasticSearchService.search(ENTITY_NAME, "test2", null, null, 0, 10, new SearchFlags().setFulltext(false)); + searchResult = _elasticSearchService.search(List.of(ENTITY_NAME), "test2", null, null, 0, 10, new SearchFlags().setFulltext(false)); assertEquals(searchResult.getNumEntities().intValue(), 1); assertEquals(searchResult.getEntities().get(0).getEntity(), urn2); browseResult = _elasticSearchService.browse(ENTITY_NAME, "", null, 0, 10); @@ -152,7 +153,7 @@ public void testElasticSearchServiceStructuredQuery() throws Exception { _elasticSearchService.deleteDocument(ENTITY_NAME, urn.toString()); _elasticSearchService.deleteDocument(ENTITY_NAME, urn2.toString()); syncAfterWrite(_bulkProcessor); - searchResult = _elasticSearchService.search(ENTITY_NAME, "test2", null, null, 0, 10, new SearchFlags().setFulltext(false)); + searchResult = _elasticSearchService.search(List.of(ENTITY_NAME), "test2", null, null, 0, 10, new SearchFlags().setFulltext(false)); assertEquals(searchResult.getNumEntities().intValue(), 0); browseResult = _elasticSearchService.browse(ENTITY_NAME, "", null, 0, 10); assertEquals(browseResult.getMetadata().getTotalNumEntities().longValue(), 0); @@ -162,7 +163,7 @@ public void testElasticSearchServiceStructuredQuery() throws Exception { @Test public void testElasticSearchServiceFulltext() throws Exception { - SearchResult searchResult = _elasticSearchService.search(ENTITY_NAME, "test", null, null, 0, 10, new SearchFlags().setFulltext(true)); + SearchResult searchResult = _elasticSearchService.search(List.of(ENTITY_NAME), "test", null, null, 0, 10, new SearchFlags().setFulltext(true)); assertEquals(searchResult.getNumEntities().intValue(), 0); Urn urn = new TestEntityUrn("test", "urn1", "VALUE_1"); @@ -175,7 +176,7 @@ public void testElasticSearchServiceFulltext() throws Exception { _elasticSearchService.upsertDocument(ENTITY_NAME, document.toString(), urn.toString()); syncAfterWrite(_bulkProcessor); - searchResult = _elasticSearchService.search(ENTITY_NAME, "test", null, null, 0, 10, new SearchFlags().setFulltext(true)); + searchResult = _elasticSearchService.search(List.of(ENTITY_NAME), "test", null, null, 0, 10, new SearchFlags().setFulltext(true)); assertEquals(searchResult.getNumEntities().intValue(), 1); assertEquals(searchResult.getEntities().get(0).getEntity(), urn); @@ -192,7 +193,7 @@ public void testElasticSearchServiceFulltext() throws Exception { _elasticSearchService.upsertDocument(ENTITY_NAME, document2.toString(), urn2.toString()); syncAfterWrite(_bulkProcessor); - searchResult = _elasticSearchService.search(ENTITY_NAME, "test2", null, null, 0, 10, new SearchFlags().setFulltext(true)); + searchResult = _elasticSearchService.search(List.of(ENTITY_NAME), "test2", null, null, 0, 10, new SearchFlags().setFulltext(true)); assertEquals(searchResult.getNumEntities().intValue(), 1); assertEquals(searchResult.getEntities().get(0).getEntity(), urn2); @@ -203,7 +204,7 @@ public void testElasticSearchServiceFulltext() throws Exception { _elasticSearchService.deleteDocument(ENTITY_NAME, urn.toString()); _elasticSearchService.deleteDocument(ENTITY_NAME, urn2.toString()); syncAfterWrite(_bulkProcessor); - searchResult = _elasticSearchService.search(ENTITY_NAME, "test2", null, null, 0, 10, new SearchFlags().setFulltext(true)); + searchResult = _elasticSearchService.search(List.of(ENTITY_NAME), "test2", null, null, 0, 10, new SearchFlags().setFulltext(true)); assertEquals(searchResult.getNumEntities().intValue(), 0); assertEquals(_elasticSearchService.docCount(ENTITY_NAME), 0); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/fixtures/SampleDataFixtureTests.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/fixtures/SampleDataFixtureTests.java index 817cf5aa5b37b..f4a8400fb005c 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/fixtures/SampleDataFixtureTests.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/fixtures/SampleDataFixtureTests.java @@ -54,6 +54,7 @@ import java.util.stream.IntStream; import java.util.stream.Stream; +import static com.linkedin.metadata.Constants.*; import static com.linkedin.metadata.ESTestUtils.*; import static com.linkedin.metadata.search.elasticsearch.query.request.SearchQueryBuilder.STRUCTURED_QUERY_PREFIX; import static com.linkedin.metadata.utils.SearchUtil.*; @@ -189,10 +190,10 @@ public void testDatasetHasTags() throws IOException { @Test public void testFixtureInitialization() { assertNotNull(searchService); - SearchResult noResult = search(searchService, "no results"); + SearchResult noResult = searchAcrossEntities(searchService, "no results"); assertEquals(0, noResult.getEntities().size()); - final SearchResult result = search(searchService, "test"); + final SearchResult result = searchAcrossEntities(searchService, "test"); Map expectedTypes = Map.of( "dataset", 13, @@ -238,7 +239,7 @@ public void testDataPlatform() { .build(); expected.forEach((key, value) -> { - SearchResult result = search(searchService, key); + SearchResult result = searchAcrossEntities(searchService, key); assertEquals(result.getEntities().size(), value.intValue(), String.format("Unexpected data platform `%s` hits.", key)); // max is 100 without pagination }); @@ -254,14 +255,14 @@ public void testUrn() { "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BOOL_LIST_feature)", "urn:li:mlModel:(urn:li:dataPlatform:science,scienceModel,PROD)" ).forEach(query -> - assertTrue(search(searchService, query).getEntities().size() >= 1, + assertTrue(searchAcrossEntities(searchService, query).getEntities().size() >= 1, String.format("Unexpected >1 urn result for `%s`", query)) ); } @Test public void testExactTable() { - SearchResult results = search(searchService, "stg_customers"); + SearchResult results = searchAcrossEntities(searchService, "stg_customers"); assertEquals(results.getEntities().size(), 1, "Unexpected single urn result for `stg_customers`"); assertEquals(results.getEntities().get(0).getEntity().toString(), "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_customers,PROD)"); @@ -278,7 +279,7 @@ public void testStemming() { testSets.forEach(testSet -> { Integer expectedResults = null; for (String testQuery : testSet) { - SearchResult results = search(searchService, testQuery); + SearchResult results = searchAcrossEntities(searchService, testQuery); assertTrue(results.hasEntities() && !results.getEntities().isEmpty(), String.format("Expected search results for `%s`", testQuery)); @@ -296,7 +297,7 @@ public void testStemmingOverride() throws IOException { Set testSet = Set.of("customer", "customers"); Set results = testSet.stream() - .map(test -> search(searchService, test)) + .map(test -> searchAcrossEntities(searchService, test)) .collect(Collectors.toSet()); results.forEach(r -> assertTrue(r.hasEntities() && !r.getEntities().isEmpty(), "Expected search results")); @@ -349,7 +350,7 @@ public void testDelimitedSynonym() throws IOException { "customer acquisition cost" ); List resultCounts = testSet.stream().map(q -> { - SearchResult result = search(searchService, q); + SearchResult result = searchAcrossEntities(searchService, q); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), "Expected search results for: " + q); return result.getEntities().size(); @@ -382,7 +383,7 @@ public void testUrnSynonym() throws IOException { "big query" ); List results = testSet.stream().map(query -> { - SearchResult result = search(searchService, query); + SearchResult result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), "Expected search results for: " + query); return result; }).collect(Collectors.toList()); @@ -621,7 +622,7 @@ public void testSmokeTestQueries() { ); Map results = expectedFulltextMinimums.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> search(searchService, entry.getKey()))); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> searchAcrossEntities(searchService, entry.getKey()))); results.forEach((key, value) -> { Integer actualCount = value.getEntities().size(); @@ -691,7 +692,7 @@ public void testUnderscore() throws IOException { @Test public void testFacets() { Set expectedFacets = Set.of("entity", "typeNames", "platform", "origin", "tags"); - SearchResult testResult = search(searchService, "cypress"); + SearchResult testResult = searchAcrossEntities(searchService, "cypress"); expectedFacets.forEach(facet -> { assertTrue(testResult.getMetadata().getAggregations().stream().anyMatch(agg -> agg.getName().equals(facet)), String.format("Failed to find facet `%s` in %s", facet, @@ -703,7 +704,7 @@ public void testFacets() { @Test public void testNestedAggregation() { Set expectedFacets = Set.of("platform"); - SearchResult testResult = search(searchService, "cypress", List.copyOf(expectedFacets)); + SearchResult testResult = searchAcrossEntities(searchService, "cypress", List.copyOf(expectedFacets)); assertEquals(testResult.getMetadata().getAggregations().size(), 1); expectedFacets.forEach(facet -> { assertTrue(testResult.getMetadata().getAggregations().stream().anyMatch(agg -> agg.getName().equals(facet)), @@ -713,7 +714,7 @@ public void testNestedAggregation() { }); expectedFacets = Set.of("platform", "typeNames", "_entityType", "entity"); - SearchResult testResult2 = search(searchService, "cypress", List.copyOf(expectedFacets)); + SearchResult testResult2 = searchAcrossEntities(searchService, "cypress", List.copyOf(expectedFacets)); assertEquals(testResult2.getMetadata().getAggregations().size(), 4); expectedFacets.forEach(facet -> { assertTrue(testResult2.getMetadata().getAggregations().stream().anyMatch(agg -> agg.getName().equals(facet)), @@ -723,11 +724,11 @@ public void testNestedAggregation() { }); String singleNestedFacet = String.format("_entityType%sowners", AGGREGATION_SEPARATOR_CHAR); expectedFacets = Set.of(singleNestedFacet); - SearchResult testResultSingleNested = search(searchService, "cypress", List.copyOf(expectedFacets)); + SearchResult testResultSingleNested = searchAcrossEntities(searchService, "cypress", List.copyOf(expectedFacets)); assertEquals(testResultSingleNested.getMetadata().getAggregations().size(), 1); expectedFacets = Set.of("platform", singleNestedFacet, "typeNames", "origin"); - SearchResult testResultNested = search(searchService, "cypress", List.copyOf(expectedFacets)); + SearchResult testResultNested = searchAcrossEntities(searchService, "cypress", List.copyOf(expectedFacets)); assertEquals(testResultNested.getMetadata().getAggregations().size(), 4); expectedFacets.forEach(facet -> { assertTrue(testResultNested.getMetadata().getAggregations().stream().anyMatch(agg -> agg.getName().equals(facet)), @@ -811,6 +812,19 @@ public void testScrollAcrossEntities() throws IOException { assertEquals(totalResults, 8); } + @Test + public void testSearchAcrossMultipleEntities() { + String query = "logging_events"; + SearchResult result = search(searchService, query); + assertEquals((int) result.getNumEntities(), 8); + result = search(searchService, List.of(DATASET_ENTITY_NAME, DATA_JOB_ENTITY_NAME), query); + assertEquals((int) result.getNumEntities(), 8); + result = search(searchService, List.of(DATASET_ENTITY_NAME), query); + assertEquals((int) result.getNumEntities(), 4); + result = search(searchService, List.of(DATA_JOB_ENTITY_NAME), query); + assertEquals((int) result.getNumEntities(), 4); + } + @Test public void testQuotedAnalyzer() throws IOException { AnalyzeRequest request = AnalyzeRequest.withIndexAnalyzer( @@ -875,7 +889,7 @@ public void testFragmentUrns() { ); testSet.forEach(query -> { - SearchResult result = search(searchService, query); + SearchResult result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected partial urn search results", query)); @@ -941,7 +955,7 @@ public void testPlatformTest() { @Test public void testStructQueryFieldMatch() { String query = STRUCTURED_QUERY_PREFIX + "name: customers"; - SearchResult result = search(searchService, query); + SearchResult result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -954,7 +968,7 @@ public void testStructQueryFieldMatch() { @Test public void testStructQueryFieldPrefixMatch() { String query = STRUCTURED_QUERY_PREFIX + "name: customers*"; - SearchResult result = search(searchService, query); + SearchResult result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -967,7 +981,7 @@ public void testStructQueryFieldPrefixMatch() { @Test public void testStructQueryCustomPropertiesKeyPrefix() { String query = STRUCTURED_QUERY_PREFIX + "customProperties: node_type=*"; - SearchResult result = search(searchService, query); + SearchResult result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -980,7 +994,7 @@ public void testStructQueryCustomPropertiesKeyPrefix() { @Test public void testStructQueryCustomPropertiesMatch() { String query = STRUCTURED_QUERY_PREFIX + "customProperties: node_type=model"; - SearchResult result = search(searchService, query); + SearchResult result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -998,7 +1012,7 @@ public void testCustomPropertiesQuoted() { ); Map results = expectedResults.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> search(searchService, entry.getKey()))); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> searchAcrossEntities(searchService, entry.getKey()))); results.forEach((key, value) -> { Integer actualCount = value.getEntities().size(); @@ -1012,7 +1026,7 @@ public void testCustomPropertiesQuoted() { @Test public void testStructQueryFieldPaths() { String query = STRUCTURED_QUERY_PREFIX + "fieldPaths: customer_id"; - SearchResult result = search(searchService, query); + SearchResult result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -1025,7 +1039,7 @@ public void testStructQueryFieldPaths() { @Test public void testStructQueryBoolean() { String query = STRUCTURED_QUERY_PREFIX + "editedFieldTags:urn\\:li\\:tag\\:Legacy OR tags:urn\\:li\\:tag\\:testTag"; - SearchResult result = search(searchService, query); + SearchResult result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -1035,7 +1049,7 @@ public void testStructQueryBoolean() { assertEquals(result.getEntities().size(), 2); query = STRUCTURED_QUERY_PREFIX + "editedFieldTags:urn\\:li\\:tag\\:Legacy"; - result = search(searchService, query); + result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -1045,7 +1059,7 @@ public void testStructQueryBoolean() { assertEquals(result.getEntities().size(), 1); query = STRUCTURED_QUERY_PREFIX + "tags:urn\\:li\\:tag\\:testTag"; - result = search(searchService, query); + result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -1058,7 +1072,7 @@ public void testStructQueryBoolean() { @Test public void testStructQueryBrowsePaths() { String query = STRUCTURED_QUERY_PREFIX + "browsePaths:*/dbt/*"; - SearchResult result = search(searchService, query); + SearchResult result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -1071,7 +1085,7 @@ public void testStructQueryBrowsePaths() { @Test public void testOr() { String query = "stg_customers | logging_events"; - SearchResult result = search(searchService, query); + SearchResult result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1079,7 +1093,7 @@ public void testOr() { assertEquals(result.getEntities().size(), 9); query = "stg_customers"; - result = search(searchService, query); + result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1087,7 +1101,7 @@ public void testOr() { assertEquals(result.getEntities().size(), 1); query = "logging_events"; - result = search(searchService, query); + result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1098,7 +1112,7 @@ public void testOr() { @Test public void testNegate() { String query = "logging_events -bckp"; - SearchResult result = search(searchService, query); + SearchResult result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1106,7 +1120,7 @@ public void testNegate() { assertEquals(result.getEntities().size(), 7); query = "logging_events"; - result = search(searchService, query); + result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1117,7 +1131,7 @@ public void testNegate() { @Test public void testPrefix() { String query = "bigquery"; - SearchResult result = search(searchService, query); + SearchResult result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1125,7 +1139,7 @@ public void testPrefix() { assertEquals(result.getEntities().size(), 8); query = "big*"; - result = search(searchService, query); + result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1136,7 +1150,7 @@ public void testPrefix() { @Test public void testParens() { String query = "dbt | (bigquery + covid19)"; - SearchResult result = search(searchService, query); + SearchResult result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1144,7 +1158,7 @@ public void testParens() { assertEquals(result.getEntities().size(), 11); query = "dbt"; - result = search(searchService, query); + result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1152,7 +1166,7 @@ public void testParens() { assertEquals(result.getEntities().size(), 9); query = "bigquery + covid19"; - result = search(searchService, query); + result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1160,7 +1174,7 @@ public void testParens() { assertEquals(result.getEntities().size(), 2); query = "bigquery"; - result = search(searchService, query); + result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1168,7 +1182,7 @@ public void testParens() { assertEquals(result.getEntities().size(), 8); query = "covid19"; - result = search(searchService, query); + result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1179,7 +1193,7 @@ public void testParens() { @Test public void testPrefixVsExact() { String query = "\"customers\""; - SearchResult result = search(searchService, query); + SearchResult result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -1197,7 +1211,7 @@ public void testPrefixVsExact() { public void testPrefixVsExactCaseSensitivity() { List insensitiveExactMatches = List.of("testExactMatchCase", "testexactmatchcase", "TESTEXACTMATCHCASE"); for (String query : insensitiveExactMatches) { - SearchResult result = search(searchService, query); + SearchResult result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); @@ -1214,7 +1228,7 @@ public void testPrefixVsExactCaseSensitivity() { @Test public void testColumnExactMatch() { String query = "unit_data"; - SearchResult result = search(searchService, query); + SearchResult result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), @@ -1227,7 +1241,7 @@ public void testColumnExactMatch() { "Expected table name exact match first"); query = "special_column_only_present_here_info"; - result = search(searchService, query); + result = searchAcrossEntities(searchService, query); assertTrue(result.hasEntities() && !result.getEntities().isEmpty(), String.format("%s - Expected search results", query)); assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()), diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/fixtures/SearchLineageDataFixtureTests.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/fixtures/SearchLineageDataFixtureTests.java index f726eba547b99..55f7d4618f479 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/fixtures/SearchLineageDataFixtureTests.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/fixtures/SearchLineageDataFixtureTests.java @@ -2,6 +2,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.metadata.ESSearchLineageFixture; +import com.linkedin.metadata.ESTestUtils; import com.linkedin.metadata.search.LineageSearchResult; import com.linkedin.metadata.search.LineageSearchService; import com.linkedin.metadata.search.SearchResult; @@ -14,7 +15,6 @@ import java.net.URISyntaxException; -import static com.linkedin.metadata.ESTestUtils.search; import static com.linkedin.metadata.ESTestUtils.lineage; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; @@ -35,10 +35,10 @@ public class SearchLineageDataFixtureTests extends AbstractTestNGSpringContextTe @Test public void testFixtureInitialization() { assertNotNull(searchService); - SearchResult noResult = search(searchService, "no results"); + SearchResult noResult = ESTestUtils.searchAcrossEntities(searchService, "no results"); assertEquals(noResult.getEntities().size(), 0); - SearchResult result = search(searchService, "e3859789eed1cef55288b44f016ee08290d9fd08973e565c112d8"); + SearchResult result = ESTestUtils.searchAcrossEntities(searchService, "e3859789eed1cef55288b44f016ee08290d9fd08973e565c112d8"); assertEquals(result.getEntities().size(), 1); assertEquals(result.getEntities().get(0).getEntity().toString(), diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHook.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHook.java index 780927d21417f..2be719ed263ea 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHook.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHook.java @@ -141,7 +141,7 @@ public void invoke(@Nonnull MetadataChangeLog event) { private void handleEntityKeyEvent(DatasetUrn datasetUrn) { Filter entitiesWithYouAsSiblingFilter = createFilterForEntitiesWithYouAsSibling(datasetUrn); final SearchResult searchResult = _searchService.search( - "dataset", + List.of(DATASET_ENTITY_NAME), "*", entitiesWithYouAsSiblingFilter, null, diff --git a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHookTest.java b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHookTest.java index 3eb21125bbaf2..5fb2cfaaef2d1 100644 --- a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHookTest.java +++ b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHookTest.java @@ -255,7 +255,7 @@ public void testInvokeWhenThereIsAKeyBeingReingested() throws Exception { Mockito.when( _mockSearchService.search( - anyString(), anyString(), any(), any(), anyInt(), anyInt(), eq(new SearchFlags().setFulltext(false) + any(), anyString(), any(), any(), anyInt(), anyInt(), eq(new SearchFlags().setFulltext(false) .setSkipAggregates(true).setSkipHighlighting(true)) )).thenReturn(returnSearchResult); diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndices.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndices.java index 486961c2c1f07..097dcfdfdf52e 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndices.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndices.java @@ -74,7 +74,7 @@ public ExecutionMode getExecutionMode() { private int getAndRestoreTermAspectIndices(int start, AuditStamp auditStamp, AspectSpec termAspectSpec) throws Exception { SearchResult termsResult = - _entitySearchService.search(Constants.GLOSSARY_TERM_ENTITY_NAME, "", null, + _entitySearchService.search(List.of(Constants.GLOSSARY_TERM_ENTITY_NAME), "", null, null, start, BATCH_SIZE, new SearchFlags().setFulltext(false) .setSkipAggregates(true).setSkipHighlighting(true)); List termUrns = termsResult.getEntities().stream().map(SearchEntity::getEntity).collect(Collectors.toList()); @@ -116,7 +116,7 @@ null, start, BATCH_SIZE, new SearchFlags().setFulltext(false) } private int getAndRestoreNodeAspectIndices(int start, AuditStamp auditStamp, AspectSpec nodeAspectSpec) throws Exception { - SearchResult nodesResult = _entitySearchService.search(Constants.GLOSSARY_NODE_ENTITY_NAME, "", + SearchResult nodesResult = _entitySearchService.search(List.of(Constants.GLOSSARY_NODE_ENTITY_NAME), "", null, null, start, BATCH_SIZE, new SearchFlags().setFulltext(false) .setSkipAggregates(true).setSkipHighlighting(true)); List nodeUrns = nodesResult.getEntities().stream().map(SearchEntity::getEntity).collect(Collectors.toList()); diff --git a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndicesTest.java b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndicesTest.java index 88c63110ee63e..d56fbed07f890 100644 --- a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndicesTest.java +++ b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndicesTest.java @@ -21,6 +21,7 @@ import com.linkedin.metadata.search.SearchResult; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.mxe.MetadataChangeProposal; +import java.util.List; import org.mockito.Mockito; import org.testng.annotations.Test; @@ -40,7 +41,7 @@ private void mockGetTermInfo(Urn glossaryTermUrn, EntitySearchService mockSearch termInfoAspects.put(Constants.GLOSSARY_TERM_INFO_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(new GlossaryTermInfo().setName("test").data()))); Map termInfoResponses = new HashMap<>(); termInfoResponses.put(glossaryTermUrn, new EntityResponse().setUrn(glossaryTermUrn).setAspects(new EnvelopedAspectMap(termInfoAspects))); - Mockito.when(mockSearchService.search(Constants.GLOSSARY_TERM_ENTITY_NAME, "", null, null, 0, 1000, + Mockito.when(mockSearchService.search(List.of(Constants.GLOSSARY_TERM_ENTITY_NAME), "", null, null, 0, 1000, new SearchFlags().setFulltext(false).setSkipAggregates(true).setSkipHighlighting(true))) .thenReturn(new SearchResult().setNumEntities(1).setEntities(new SearchEntityArray(ImmutableList.of(new SearchEntity().setEntity(glossaryTermUrn))))); Mockito.when(mockService.getEntitiesV2( @@ -55,7 +56,7 @@ private void mockGetNodeInfo(Urn glossaryNodeUrn, EntitySearchService mockSearch nodeInfoAspects.put(Constants.GLOSSARY_NODE_INFO_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(new GlossaryNodeInfo().setName("test").data()))); Map nodeInfoResponses = new HashMap<>(); nodeInfoResponses.put(glossaryNodeUrn, new EntityResponse().setUrn(glossaryNodeUrn).setAspects(new EnvelopedAspectMap(nodeInfoAspects))); - Mockito.when(mockSearchService.search(Constants.GLOSSARY_NODE_ENTITY_NAME, "", null, null, 0, 1000, + Mockito.when(mockSearchService.search(List.of(Constants.GLOSSARY_NODE_ENTITY_NAME), "", null, null, 0, 1000, new SearchFlags().setFulltext(false).setSkipAggregates(true).setSkipHighlighting(true))) .thenReturn(new SearchResult().setNumEntities(1).setEntities(new SearchEntityArray(ImmutableList.of(new SearchEntity().setEntity(glossaryNodeUrn))))); Mockito.when(mockService.getEntitiesV2( @@ -221,10 +222,10 @@ public void testDoesNotRunWhenAlreadyExecuted() throws Exception { Mockito.verify(mockRegistry, Mockito.times(0)).getEntitySpec(Constants.GLOSSARY_TERM_ENTITY_NAME); Mockito.verify(mockRegistry, Mockito.times(0)).getEntitySpec(Constants.GLOSSARY_NODE_ENTITY_NAME); - Mockito.verify(mockSearchService, Mockito.times(0)).search(Constants.GLOSSARY_TERM_ENTITY_NAME, + Mockito.verify(mockSearchService, Mockito.times(0)).search(List.of(Constants.GLOSSARY_TERM_ENTITY_NAME), "", null, null, 0, 1000, new SearchFlags().setFulltext(false) .setSkipAggregates(true).setSkipHighlighting(true)); - Mockito.verify(mockSearchService, Mockito.times(0)).search(Constants.GLOSSARY_NODE_ENTITY_NAME, + Mockito.verify(mockSearchService, Mockito.times(0)).search(List.of(Constants.GLOSSARY_NODE_ENTITY_NAME), "", null, null, 0, 1000, new SearchFlags().setFulltext(false) .setSkipAggregates(true).setSkipHighlighting(true)); Mockito.verify(mockService, Mockito.times(0)).ingestProposal( diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java index b8fd785eaad0f..f6dedfb9a07c6 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java @@ -332,7 +332,7 @@ public Task search(@ActionParam(PARAM_ENTITY) @Nonnull String enti () -> { final SearchResult result; // This API is not used by the frontend for search bars so we default to structured - result = _entitySearchService.search(entityName, input, filter, sortCriterion, start, count, searchFlags); + result = _entitySearchService.search(List.of(entityName), input, filter, sortCriterion, start, count, searchFlags); return validateSearchResult(result, _entityService); }, MetricRegistry.name(this.getClass(), "search")); diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java index de5bdb62f201b..a46b58aabfb0b 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java @@ -64,7 +64,7 @@ public interface EntitySearchService { * Safe for non-structured, user input, queries with an attempt to provide some advanced features * Impl * - * @param entityName name of the entity + * @param entityNames names of the entities * @param input the search input text * @param postFilters the request map with fields and values as filters to be applied to search hits * @param sortCriterion {@link SortCriterion} to be applied to search results @@ -74,7 +74,7 @@ public interface EntitySearchService { * @return a {@link SearchResult} that contains a list of matched documents and related search result metadata */ @Nonnull - SearchResult search(@Nonnull String entityName, @Nonnull String input, @Nullable Filter postFilters, + SearchResult search(@Nonnull List entityNames, @Nonnull String input, @Nullable Filter postFilters, @Nullable SortCriterion sortCriterion, int from, int size, @Nullable SearchFlags searchFlags); /** @@ -84,7 +84,7 @@ SearchResult search(@Nonnull String entityName, @Nonnull String input, @Nullable * Safe for non-structured, user input, queries with an attempt to provide some advanced features * Impl * - * @param entityName name of the entity + * @param entityNames names of the entities * @param input the search input text * @param postFilters the request map with fields and values as filters to be applied to search hits * @param sortCriterion {@link SortCriterion} to be applied to search results @@ -95,7 +95,7 @@ SearchResult search(@Nonnull String entityName, @Nonnull String input, @Nullable * @return a {@link SearchResult} that contains a list of matched documents and related search result metadata */ @Nonnull - SearchResult search(@Nonnull String entityName, @Nonnull String input, @Nullable Filter postFilters, + SearchResult search(@Nonnull List entityNames, @Nonnull String input, @Nullable Filter postFilters, @Nullable SortCriterion sortCriterion, int from, int size, @Nullable SearchFlags searchFlags, @Nullable List facets); /** diff --git a/mock-entity-registry/src/main/java/mock/MockEntitySpec.java b/mock-entity-registry/src/main/java/mock/MockEntitySpec.java index 097e0845504a1..f43c1f7fd6613 100644 --- a/mock-entity-registry/src/main/java/mock/MockEntitySpec.java +++ b/mock-entity-registry/src/main/java/mock/MockEntitySpec.java @@ -25,6 +25,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static com.linkedin.metadata.Constants.*; @@ -88,7 +89,8 @@ public AspectSpec createAspectSpec(T type, String nam @Override public List getAspectSpecs() { - return Collections.emptyList(); + return ASPECT_TYPE_MAP.keySet().stream().map(name -> createAspectSpec(ASPECT_TYPE_MAP.get(name), name)).collect( + Collectors.toList()); } @Override From bf9f380350db5967634638f089c78a516dfa2b6a Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Tue, 25 Jul 2023 02:05:34 -0400 Subject: [PATCH 177/222] fix(ingest): Generate browse paths v2 for more sources; properly pass platform_instance (#8501) --- .../src/datahub/ingestion/api/source.py | 2 +- .../datahub/ingestion/source/aws/sagemaker.py | 4 - .../datahub/ingestion/source/csv_enricher.py | 4 - .../ingestion/source/delta_lake/source.py | 4 - .../ingestion/source/elastic_search.py | 4 - .../src/datahub/ingestion/source/feast.py | 5 - .../src/datahub/ingestion/source/metabase.py | 4 - .../source/metadata/business_glossary.py | 14 +- .../ingestion/source/metadata/lineage.py | 15 +- .../src/datahub/ingestion/source/mode.py | 4 - .../src/datahub/ingestion/source/mongodb.py | 8 - .../src/datahub/ingestion/source/nifi.py | 4 - .../src/datahub/ingestion/source/openapi.py | 4 - .../powerbi_report_server/report_server.py | 4 - .../src/datahub/ingestion/source/redash.py | 4 - .../datahub/ingestion/source/salesforce.py | 2 +- .../datahub/ingestion/source/sql/vertica.py | 4 - .../source/usage/clickhouse_usage.py | 4 - .../ingestion/source/usage/redshift_usage.py | 4 - .../source/usage/starburst_trino_usage.py | 4 - .../datahub/testing/compare_metadata_json.py | 17 +- .../src/datahub/testing/mcp_diff.py | 6 +- .../clickhouse/clickhouse_mces_golden.json | 77 +- .../csv-enricher/csv_enricher_golden.json | 1964 +++++---- .../delta_lake_minio_mces_golden.json | 85 +- .../local/golden_mces_allow_table.json | 344 ++ .../local/golden_mces_inner_table.json | 303 ++ .../local/golden_mces_relative_path.json | 35 + .../local/golden_mces_single_table.json | 155 + .../feast/feast_repository_mces_golden.json | 112 +- .../metabase/metabase_mces_golden.json | 142 +- .../integration/mode/mode_mces_golden.json | 144 +- .../mongodb/mongodb_mces_golden.json | 60 + .../nifi/nifi_mces_golden_cluster.json | 210 + .../nifi/nifi_mces_golden_standalone.json | 75 + .../openapi/openapi_mces_golden.json | 30 + .../golden_test_platform_instance_ingest.json | 12 + .../golden_test_fail_api_ingest.json | 68 + .../golden_test_ingest.json | 102 + .../remote/golden/remote_enricher_golden.json | 30 + .../remote/golden/remote_lineage_golden.json | 40 +- .../golden_mces_file_without_extension.json | 43 +- .../local/golden_mces_multiple_files.json | 63 +- .../golden_mces_file_without_extension.json | 27 +- .../s3/golden_mces_multiple_files.json | 47 +- .../salesforce/salesforce_mces_golden.json | 60 + .../trino_usages_golden.json | 57 +- ...au_with_platform_instance_mces_golden.json | 274 +- .../unit/sagemaker/sagemaker_mces_golden.json | 3774 +++++++++-------- 49 files changed, 5530 insertions(+), 2928 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/api/source.py b/metadata-ingestion/src/datahub/ingestion/api/source.py index fa037f8d7328b..0bcc220cad49b 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/source.py +++ b/metadata-ingestion/src/datahub/ingestion/api/source.py @@ -251,7 +251,7 @@ def _get_browse_path_processor(self, dry_run: bool) -> MetadataWorkUnitProcessor platform_instance: Optional[str] = None if isinstance(config, PlatformInstanceConfigMixin) and config.platform_instance: - platform_instance = platform_instance + platform_instance = config.platform_instance return partial( auto_browse_path_v2, diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker.py b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker.py index f360b503b640f..6f6e8bbc05661 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker.py @@ -11,7 +11,6 @@ support_status, ) from datahub.ingestion.api.source import Source -from datahub.ingestion.api.source_helpers import auto_workunit_reporter from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.aws.sagemaker_processors.common import ( SagemakerSourceConfig, @@ -57,9 +56,6 @@ def create(cls, config_dict, ctx): config = SagemakerSourceConfig.parse_obj(config_dict) return cls(config, ctx) - def get_workunits(self) -> Iterable[MetadataWorkUnit]: - return auto_workunit_reporter(self.report, self.get_workunits_internal()) - def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: # get common lineage graph lineage_processor = LineageProcessor( diff --git a/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py b/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py index 1e9879f599731..e41c02b462662 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py +++ b/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py @@ -17,7 +17,6 @@ support_status, ) from datahub.ingestion.api.source import Source, SourceReport -from datahub.ingestion.api.source_helpers import auto_workunit_reporter from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source_config.csv_enricher import CSVEnricherConfig from datahub.metadata.schema_classes import ( @@ -590,9 +589,6 @@ def maybe_extract_owners( ] return owners - def get_workunits(self) -> Iterable[MetadataWorkUnit]: - return auto_workunit_reporter(self.report, self.get_workunits_internal()) - def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: # As per https://stackoverflow.com/a/49150749/5004662, we want to use # the 'utf-8-sig' encoding to handle any BOM character that may be diff --git a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py index 0c0cbaf3b4803..180ef00459214 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py @@ -21,7 +21,6 @@ support_status, ) from datahub.ingestion.api.source import Source, SourceReport -from datahub.ingestion.api.source_helpers import auto_workunit_reporter from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.aws.s3_boto_utils import get_s3_tags from datahub.ingestion.source.aws.s3_util import ( @@ -340,9 +339,6 @@ def local_get_folders(self, path: str) -> Iterable[str]: for folder in os.listdir(path): yield os.path.join(path, folder) - def get_workunits(self) -> Iterable[MetadataWorkUnit]: - return auto_workunit_reporter(self.report, self.get_workunits_internal()) - def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: self.container_WU_creator = ContainerWUCreator( self.source_config.platform, diff --git a/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py b/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py index 7fac85627ed8b..b9165ce0ed160 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py +++ b/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py @@ -33,7 +33,6 @@ support_status, ) from datahub.ingestion.api.source import Source, SourceReport -from datahub.ingestion.api.source_helpers import auto_workunit_reporter from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.common.subtypes import DatasetSubTypes from datahub.metadata.com.linkedin.pegasus2avro.common import StatusClass @@ -352,9 +351,6 @@ def create( config = ElasticsearchSourceConfig.parse_obj(config_dict) return cls(config, ctx) - def get_workunits(self) -> Iterable[MetadataWorkUnit]: - return auto_workunit_reporter(self.report, self.get_workunits_internal()) - def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: indices = self.client.indices.get_alias() for index in indices: diff --git a/metadata-ingestion/src/datahub/ingestion/source/feast.py b/metadata-ingestion/src/datahub/ingestion/source/feast.py index c4219fdc544f3..8faba7d113372 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/feast.py +++ b/metadata-ingestion/src/datahub/ingestion/source/feast.py @@ -1,7 +1,5 @@ import sys -from datahub.ingestion.api.source_helpers import auto_workunit_reporter - if sys.version_info < (3, 8): raise ImportError("Feast is only supported on Python 3.8+") @@ -370,9 +368,6 @@ def create(cls, config_dict, ctx): config = FeastRepositorySourceConfig.parse_obj(config_dict) return cls(config, ctx) - def get_workunits(self) -> Iterable[MetadataWorkUnit]: - return auto_workunit_reporter(self.report, self.get_workunits_internal()) - def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: for feature_view in self.feature_store.list_feature_views(): for entity_name in feature_view.entities: diff --git a/metadata-ingestion/src/datahub/ingestion/source/metabase.py b/metadata-ingestion/src/datahub/ingestion/source/metabase.py index 17b53cb64caf6..0a6d8c605688a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/metabase.py +++ b/metadata-ingestion/src/datahub/ingestion/source/metabase.py @@ -21,7 +21,6 @@ support_status, ) from datahub.ingestion.api.source import Source, SourceReport -from datahub.ingestion.api.source_helpers import auto_workunit_reporter from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.metadata.com.linkedin.pegasus2avro.common import ( AuditStamp, @@ -611,9 +610,6 @@ def create(cls, config_dict: dict, ctx: PipelineContext) -> Source: config = MetabaseConfig.parse_obj(config_dict) return cls(ctx, config) - def get_workunits(self) -> Iterable[MetadataWorkUnit]: - return auto_workunit_reporter(self.report, self.get_workunits_internal()) - def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: yield from self.emit_card_mces() yield from self.emit_dashboard_mces() diff --git a/metadata-ingestion/src/datahub/ingestion/source/metadata/business_glossary.py b/metadata-ingestion/src/datahub/ingestion/source/metadata/business_glossary.py index 53822994506e3..b5d9d96354fc5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/metadata/business_glossary.py +++ b/metadata-ingestion/src/datahub/ingestion/source/metadata/business_glossary.py @@ -20,11 +20,7 @@ support_status, ) from datahub.ingestion.api.source import Source, SourceReport -from datahub.ingestion.api.source_helpers import ( - auto_status_aspect, - auto_workunit, - auto_workunit_reporter, -) +from datahub.ingestion.api.source_helpers import auto_workunit from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.graph.client import DataHubGraph from datahub.utilities.registries.domain_registry import DomainRegistry @@ -503,14 +499,6 @@ def load_glossary_config( glossary_cfg = BusinessGlossaryConfig.parse_obj(config) return glossary_cfg - def get_workunits(self) -> Iterable[MetadataWorkUnit]: - return auto_workunit_reporter( - self.report, - auto_status_aspect( - self.get_workunits_internal(), - ), - ) - def get_workunits_internal( self, ) -> Iterable[MetadataWorkUnit]: diff --git a/metadata-ingestion/src/datahub/ingestion/source/metadata/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/metadata/lineage.py index 207e607ed14ca..1c0c809c16a60 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/metadata/lineage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/metadata/lineage.py @@ -1,5 +1,6 @@ import logging from dataclasses import dataclass, field +from functools import partial from typing import Any, Dict, Iterable, List, Optional from pydantic import validator @@ -26,8 +27,11 @@ platform_name, support_status, ) -from datahub.ingestion.api.source import Source, SourceReport -from datahub.ingestion.api.source_helpers import auto_workunit_reporter +from datahub.ingestion.api.source import MetadataWorkUnitProcessor, Source, SourceReport +from datahub.ingestion.api.source_helpers import ( + auto_status_aspect, + auto_workunit_reporter, +) from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.metadata.com.linkedin.pegasus2avro.dataset import ( FineGrainedLineageDownstreamType, @@ -139,8 +143,11 @@ def load_lineage_config(file_name: str) -> LineageConfig: lineage_config = LineageConfig.parse_obj(config) return lineage_config - def get_workunits(self) -> Iterable[MetadataWorkUnit]: - return auto_workunit_reporter(self.report, self.get_workunits_internal()) + def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: + return [ + auto_status_aspect, + partial(auto_workunit_reporter, self.get_report()), + ] def get_workunits_internal( self, diff --git a/metadata-ingestion/src/datahub/ingestion/source/mode.py b/metadata-ingestion/src/datahub/ingestion/source/mode.py index 2bb185619d26c..0cf9932ba0878 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/mode.py +++ b/metadata-ingestion/src/datahub/ingestion/source/mode.py @@ -25,7 +25,6 @@ support_status, ) from datahub.ingestion.api.source import Source, SourceReport -from datahub.ingestion.api.source_helpers import auto_workunit_reporter from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.metadata.com.linkedin.pegasus2avro.common import ( AuditStamp, @@ -797,9 +796,6 @@ def create(cls, config_dict: dict, ctx: PipelineContext) -> Source: config = ModeConfig.parse_obj(config_dict) return cls(ctx, config) - def get_workunits(self) -> Iterable[MetadataWorkUnit]: - return auto_workunit_reporter(self.report, self.get_workunits_internal()) - def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: yield from self.emit_dashboard_mces() yield from self.emit_chart_mces() diff --git a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py index 53e9094ee8178..f02b6845e40b5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py +++ b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py @@ -2,10 +2,6 @@ from dataclasses import dataclass, field from typing import Dict, Iterable, List, Optional, Tuple, Type, Union, ValuesView -import bson -import bson.dbref -import bson.int64 -import bson.objectid import bson.timestamp import pymongo import pymongo.collection @@ -26,7 +22,6 @@ support_status, ) from datahub.ingestion.api.source import Source, SourceReport -from datahub.ingestion.api.source_helpers import auto_workunit_reporter from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.schema_inference.object import ( SchemaDescription, @@ -301,9 +296,6 @@ def get_field_type( return SchemaFieldDataType(type=TypeClass()) - def get_workunits(self) -> Iterable[MetadataWorkUnit]: - return auto_workunit_reporter(self.report, self.get_workunits_internal()) - def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: platform = "mongodb" diff --git a/metadata-ingestion/src/datahub/ingestion/source/nifi.py b/metadata-ingestion/src/datahub/ingestion/source/nifi.py index 559d103aa6e5a..ac1e03812db3b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/nifi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/nifi.py @@ -31,7 +31,6 @@ support_status, ) from datahub.ingestion.api.source import Source, SourceReport -from datahub.ingestion.api.source_helpers import auto_workunit_reporter from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.metadata.schema_classes import ( DataFlowInfoClass, @@ -1024,9 +1023,6 @@ def authenticate(self): token_response.raise_for_status() self.session.headers.update({"Authorization": "Bearer " + token_response.text}) - def get_workunits(self) -> Iterable[MetadataWorkUnit]: - return auto_workunit_reporter(self.report, self.get_workunits_internal()) - def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: try: self.authenticate() diff --git a/metadata-ingestion/src/datahub/ingestion/source/openapi.py b/metadata-ingestion/src/datahub/ingestion/source/openapi.py index ad9aec927832b..78570a2a4ceca 100755 --- a/metadata-ingestion/src/datahub/ingestion/source/openapi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/openapi.py @@ -18,7 +18,6 @@ support_status, ) from datahub.ingestion.api.source import Source, SourceReport -from datahub.ingestion.api.source_helpers import auto_workunit_reporter from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.openapi_parser import ( clean_url, @@ -213,9 +212,6 @@ def build_wu( mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot) return ApiWorkUnit(id=dataset_name, mce=mce) - def get_workunits(self) -> Iterable[MetadataWorkUnit]: - return auto_workunit_reporter(self.report, self.get_workunits_internal()) - def get_workunits_internal(self) -> Iterable[ApiWorkUnit]: # noqa: C901 config = self.config diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py index e66119f6e8d76..80367558251a5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py @@ -26,7 +26,6 @@ support_status, ) from datahub.ingestion.api.source import Source, SourceReport -from datahub.ingestion.api.source_helpers import auto_workunit_reporter from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.powerbi_report_server.constants import ( API_ENDPOINTS, @@ -533,9 +532,6 @@ def create(cls, config_dict, ctx): config = PowerBiReportServerDashboardSourceConfig.parse_obj(config_dict) return cls(config, ctx) - def get_workunits(self) -> Iterable[MetadataWorkUnit]: - return auto_workunit_reporter(self.report, self.get_workunits_internal()) - def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: """ Datahub Ingestion framework invoke this method diff --git a/metadata-ingestion/src/datahub/ingestion/source/redash.py b/metadata-ingestion/src/datahub/ingestion/source/redash.py index bc5225509343b..4019436bda2f0 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redash.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redash.py @@ -24,7 +24,6 @@ ) from datahub.ingestion.api.registry import import_path from datahub.ingestion.api.source import Source, SourceReport -from datahub.ingestion.api.source_helpers import auto_workunit_reporter from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.metadata.com.linkedin.pegasus2avro.common import ( AuditStamp, @@ -776,9 +775,6 @@ def _emit_chart_mces(self) -> Iterable[MetadataWorkUnit]: def add_config_to_report(self) -> None: self.report.api_page_limit = self.config.api_page_limit - def get_workunits(self) -> Iterable[MetadataWorkUnit]: - return auto_workunit_reporter(self.report, self.get_workunits_internal()) - def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: self.validate_connection() self.add_config_to_report() diff --git a/metadata-ingestion/src/datahub/ingestion/source/salesforce.py b/metadata-ingestion/src/datahub/ingestion/source/salesforce.py index 8d35633010a3e..75f9a0fc34708 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/salesforce.py +++ b/metadata-ingestion/src/datahub/ingestion/source/salesforce.py @@ -281,7 +281,7 @@ def __init__(self, config: SalesforceConfig, ctx: PipelineContext) -> None: ) ) - def get_workunits(self) -> Iterable[MetadataWorkUnit]: + def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: sObjects = self.get_salesforce_objects() for sObject in sObjects: diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py index ac08593d97e28..37ccc1d1fedb5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py @@ -27,7 +27,6 @@ platform_name, support_status, ) -from datahub.ingestion.api.source_helpers import auto_workunit_reporter from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.sql.sql_common import ( SQLAlchemySource, @@ -144,9 +143,6 @@ def create(cls, config_dict: Dict, ctx: PipelineContext) -> "VerticaSource": config = VerticaConfig.parse_obj(config_dict) return cls(config, ctx) - def get_workunits(self) -> Iterable[MetadataWorkUnit]: - return auto_workunit_reporter(self.report, self.get_workunits_internal()) - def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]: sql_config = self.config if logger.isEnabledFor(logging.DEBUG): diff --git a/metadata-ingestion/src/datahub/ingestion/source/usage/clickhouse_usage.py b/metadata-ingestion/src/datahub/ingestion/source/usage/clickhouse_usage.py index 412ce36170048..ffa08752070dd 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/usage/clickhouse_usage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/usage/clickhouse_usage.py @@ -22,7 +22,6 @@ support_status, ) from datahub.ingestion.api.source import Source, SourceReport -from datahub.ingestion.api.source_helpers import auto_workunit_reporter from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.sql.clickhouse import ClickHouseConfig from datahub.ingestion.source.usage.usage_common import ( @@ -111,9 +110,6 @@ def create(cls, config_dict, ctx): config = ClickHouseUsageConfig.parse_obj(config_dict) return cls(ctx, config) - def get_workunits(self) -> Iterable[MetadataWorkUnit]: - return auto_workunit_reporter(self.report, self.get_workunits_internal()) - def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: """Gets ClickHouse usage stats as work units""" access_events = self._get_clickhouse_history() diff --git a/metadata-ingestion/src/datahub/ingestion/source/usage/redshift_usage.py b/metadata-ingestion/src/datahub/ingestion/source/usage/redshift_usage.py index 0aa4efb47ad39..ea817f40f6a2b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/usage/redshift_usage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/usage/redshift_usage.py @@ -24,7 +24,6 @@ support_status, ) from datahub.ingestion.api.source import Source, SourceReport -from datahub.ingestion.api.source_helpers import auto_workunit_reporter from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.sql.redshift import RedshiftConfig from datahub.ingestion.source.usage.usage_common import ( @@ -218,9 +217,6 @@ def create(cls, config_dict: Dict, ctx: PipelineContext) -> "RedshiftUsageSource config = RedshiftUsageConfig.parse_obj(config_dict) return cls(config, ctx) - def get_workunits(self) -> Iterable[MetadataWorkUnit]: - return auto_workunit_reporter(self.report, self.get_workunits_internal()) - def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: """Gets Redshift usage stats as work units""" engine: Engine = self._make_sql_engine() diff --git a/metadata-ingestion/src/datahub/ingestion/source/usage/starburst_trino_usage.py b/metadata-ingestion/src/datahub/ingestion/source/usage/starburst_trino_usage.py index 49f64f3bff661..7dd66fd1e3d0c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/usage/starburst_trino_usage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/usage/starburst_trino_usage.py @@ -21,7 +21,6 @@ support_status, ) from datahub.ingestion.api.source import Source, SourceReport -from datahub.ingestion.api.source_helpers import auto_workunit_reporter from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.sql.trino import TrinoConfig from datahub.ingestion.source.usage.usage_common import ( @@ -130,9 +129,6 @@ def create(cls, config_dict, ctx): config = TrinoUsageConfig.parse_obj(config_dict) return cls(ctx, config) - def get_workunits(self) -> Iterable[MetadataWorkUnit]: - return auto_workunit_reporter(self.report, self.get_workunits_internal()) - def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: access_events = self._get_trino_history() # If the query results is empty, we don't want to proceed diff --git a/metadata-ingestion/src/datahub/testing/compare_metadata_json.py b/metadata-ingestion/src/datahub/testing/compare_metadata_json.py index 5b65ba6006838..466e652277fd1 100644 --- a/metadata-ingestion/src/datahub/testing/compare_metadata_json.py +++ b/metadata-ingestion/src/datahub/testing/compare_metadata_json.py @@ -14,7 +14,7 @@ from datahub.ingestion.sink.file import write_metadata_file from datahub.ingestion.source.file import read_metadata_file -from datahub.testing.mcp_diff import MCPDiff, get_aspects_by_urn +from datahub.testing.mcp_diff import CannotCompareMCPs, MCPDiff, get_aspects_by_urn logger = logging.getLogger(__name__) @@ -98,12 +98,15 @@ def diff_metadata_json( output=output_map, ignore_paths=ignore_paths, ) + except CannotCompareMCPs as e: + logger.info(f"{e}, falling back to MCE diff") except AssertionError as e: logger.warning(f"Reverting to old diff method: {e}") logger.debug("Error with new diff method", exc_info=True) - return DeepDiff( - golden, - output, - exclude_regex_paths=ignore_paths, - ignore_order=True, - ) + + return DeepDiff( + golden, + output, + exclude_regex_paths=ignore_paths, + ignore_order=True, + ) diff --git a/metadata-ingestion/src/datahub/testing/mcp_diff.py b/metadata-ingestion/src/datahub/testing/mcp_diff.py index c253e56a3a707..eb1d9371cc3c4 100644 --- a/metadata-ingestion/src/datahub/testing/mcp_diff.py +++ b/metadata-ingestion/src/datahub/testing/mcp_diff.py @@ -83,6 +83,10 @@ def give_up_diffing(self, *args: Any, **kwargs: Any) -> bool: AspectsByUrn = Dict[str, Dict[str, List[AspectForDiff]]] +class CannotCompareMCPs(Exception): + pass + + def get_aspects_by_urn(obj: object) -> AspectsByUrn: """Restructure a list of serialized MCPs by urn and aspect. Retains information like the original dict and index to facilitate `apply_delta` later. @@ -95,7 +99,7 @@ def get_aspects_by_urn(obj: object) -> AspectsByUrn: for i, entry in enumerate(obj): assert isinstance(entry, dict), entry if "proposedSnapshot" in entry: - raise AssertionError("Found MCEs in output") + raise CannotCompareMCPs("Found MCEs") elif "entityUrn" in entry and "aspectName" in entry and "aspect" in entry: urn = entry["entityUrn"] aspect_name = entry["aspectName"] diff --git a/metadata-ingestion/tests/integration/clickhouse/clickhouse_mces_golden.json b/metadata-ingestion/tests/integration/clickhouse/clickhouse_mces_golden.json index 67915ca6be713..6dd18f9403188 100644 --- a/metadata-ingestion/tests/integration/clickhouse/clickhouse_mces_golden.json +++ b/metadata-ingestion/tests/integration/clickhouse/clickhouse_mces_golden.json @@ -75,7 +75,12 @@ "aspectName": "browsePathsV2", "aspect": { "json": { - "path": [] + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:clickhouse,clickhousetestserver)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:clickhouse,clickhousetestserver)" + } + ] } }, "systemMetadata": { @@ -176,6 +181,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:clickhouse,clickhousetestserver)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:clickhouse,clickhousetestserver)" + }, { "id": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c", "urn": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c" @@ -246,11 +255,11 @@ "primary_key": "col_Int64", "sampling_key": "", "storage_policy": "default", - "metadata_modification_time": "2023-07-03 18:52:30", + "metadata_modification_time": "2023-07-24 21:34:01", "total_rows": "10", "total_bytes": "671", - "data_paths": "['/var/lib/clickhouse/store/2d4/2d43771f-2a9f-4a28-962e-992a7c08102f/']", - "metadata_path": "/var/lib/clickhouse/store/e2e/e2e3221c-2ffa-4009-b71e-ad306b778310/mv_target_table.sql" + "data_paths": "['/var/lib/clickhouse/store/6cb/6cbac4d1-c700-4f8a-9cc9-542cc349e497/']", + "metadata_path": "/var/lib/clickhouse/store/0a2/0a2bd3dd-893f-4f9a-b310-92e4c830091a/mv_target_table.sql" }, "name": "mv_target_table", "description": "This is target table for materialized view", @@ -394,6 +403,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:clickhouse,clickhousetestserver)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:clickhouse,clickhousetestserver)" + }, { "id": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c", "urn": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c" @@ -444,11 +457,11 @@ "primary_key": "", "sampling_key": "", "storage_policy": "default", - "metadata_modification_time": "2023-07-03 18:52:30", + "metadata_modification_time": "2023-07-24 21:34:01", "total_rows": "0", "total_bytes": "0", - "data_paths": "['/var/lib/clickhouse/store/700/70013972-f4ad-4c8b-a4bd-b397c9cc727f/']", - "metadata_path": "/var/lib/clickhouse/store/e2e/e2e3221c-2ffa-4009-b71e-ad306b778310/test_data_types.sql" + "data_paths": "['/var/lib/clickhouse/store/339/339ddf61-6dc4-47ae-9ae5-a358864e6457/']", + "metadata_path": "/var/lib/clickhouse/store/0a2/0a2bd3dd-893f-4f9a-b310-92e4c830091a/test_data_types.sql" }, "name": "test_data_types", "description": "This table has basic types", @@ -1021,6 +1034,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:clickhouse,clickhousetestserver)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:clickhouse,clickhousetestserver)" + }, { "id": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c", "urn": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c" @@ -1095,11 +1112,11 @@ "primary_key": "", "sampling_key": "", "storage_policy": "", - "metadata_modification_time": "2023-07-03 18:52:30", + "metadata_modification_time": "2023-07-24 21:34:01", "total_rows": "None", "total_bytes": "None", "data_paths": "[]", - "metadata_path": "/var/lib/clickhouse/store/e2e/e2e3221c-2ffa-4009-b71e-ad306b778310/test_dict.sql" + "metadata_path": "/var/lib/clickhouse/store/0a2/0a2bd3dd-893f-4f9a-b310-92e4c830091a/test_dict.sql" }, "name": "test_dict", "description": "", @@ -1204,6 +1221,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:clickhouse,clickhousetestserver)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:clickhouse,clickhousetestserver)" + }, { "id": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c", "urn": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c" @@ -1254,11 +1275,11 @@ "primary_key": "", "sampling_key": "", "storage_policy": "default", - "metadata_modification_time": "2023-07-03 18:52:30", + "metadata_modification_time": "2023-07-24 21:34:01", "total_rows": "0", "total_bytes": "0", - "data_paths": "['/var/lib/clickhouse/store/04c/04c9735e-af23-4850-a034-8da8f03d75af/']", - "metadata_path": "/var/lib/clickhouse/store/e2e/e2e3221c-2ffa-4009-b71e-ad306b778310/test_nested_data_types.sql" + "data_paths": "['/var/lib/clickhouse/store/22c/22c46b00-4f2a-444a-8bee-73e60b9deba6/']", + "metadata_path": "/var/lib/clickhouse/store/0a2/0a2bd3dd-893f-4f9a-b310-92e4c830091a/test_nested_data_types.sql" }, "name": "test_nested_data_types", "description": "This table has nested types", @@ -1467,6 +1488,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:clickhouse,clickhousetestserver)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:clickhouse,clickhousetestserver)" + }, { "id": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c", "urn": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c" @@ -1541,11 +1566,11 @@ "primary_key": "", "sampling_key": "", "storage_policy": "", - "metadata_modification_time": "2023-07-03 18:52:30", + "metadata_modification_time": "2023-07-24 21:34:01", "total_rows": "None", "total_bytes": "None", - "data_paths": "['/var/lib/clickhouse/store/2d4/2d43771f-2a9f-4a28-962e-992a7c08102f/']", - "metadata_path": "/var/lib/clickhouse/store/e2e/e2e3221c-2ffa-4009-b71e-ad306b778310/mv_with_target_table.sql", + "data_paths": "['/var/lib/clickhouse/store/6cb/6cbac4d1-c700-4f8a-9cc9-542cc349e497/']", + "metadata_path": "/var/lib/clickhouse/store/0a2/0a2bd3dd-893f-4f9a-b310-92e4c830091a/mv_with_target_table.sql", "view_definition": "", "is_view": "True" }, @@ -1708,6 +1733,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:clickhouse,clickhousetestserver)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:clickhouse,clickhousetestserver)" + }, { "id": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c", "urn": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c" @@ -1782,11 +1811,11 @@ "primary_key": "", "sampling_key": "", "storage_policy": "", - "metadata_modification_time": "2023-07-03 18:52:30", + "metadata_modification_time": "2023-07-24 21:34:01", "total_rows": "0", "total_bytes": "0", - "data_paths": "['/var/lib/clickhouse/store/649/64934dd8-0347-4e30-a604-8b27022dc799/']", - "metadata_path": "/var/lib/clickhouse/store/e2e/e2e3221c-2ffa-4009-b71e-ad306b778310/mv_without_target_table.sql", + "data_paths": "['/var/lib/clickhouse/store/0f1/0f172bf3-80e5-4ba7-9ae3-938da0a9799d/']", + "metadata_path": "/var/lib/clickhouse/store/0a2/0a2bd3dd-893f-4f9a-b310-92e4c830091a/mv_without_target_table.sql", "view_definition": "", "is_view": "True" }, @@ -1949,6 +1978,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:clickhouse,clickhousetestserver)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:clickhouse,clickhousetestserver)" + }, { "id": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c", "urn": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c" @@ -2023,11 +2056,11 @@ "primary_key": "", "sampling_key": "", "storage_policy": "", - "metadata_modification_time": "2023-07-03 18:52:30", + "metadata_modification_time": "2023-07-24 21:34:01", "total_rows": "None", "total_bytes": "None", "data_paths": "[]", - "metadata_path": "/var/lib/clickhouse/store/e2e/e2e3221c-2ffa-4009-b71e-ad306b778310/test_view.sql", + "metadata_path": "/var/lib/clickhouse/store/0a2/0a2bd3dd-893f-4f9a-b310-92e4c830091a/test_view.sql", "view_definition": "", "is_view": "True" }, @@ -2138,6 +2171,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:clickhouse,clickhousetestserver)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:clickhouse,clickhousetestserver)" + }, { "id": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c", "urn": "urn:li:container:ab016b94aa0d75c5b9205c33260e989c" diff --git a/metadata-ingestion/tests/integration/csv-enricher/csv_enricher_golden.json b/metadata-ingestion/tests/integration/csv-enricher/csv_enricher_golden.json index fa2e0cf6570b5..d56eb50843dd0 100644 --- a/metadata-ingestion/tests/integration/csv-enricher/csv_enricher_golden.json +++ b/metadata-ingestion/tests/integration/csv-enricher/csv_enricher_golden.json @@ -1,932 +1,1142 @@ [ - { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", - "changeType": "UPSERT", - "aspectName": "glossaryTerms", - "aspect": { - "json": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CustomerAccount" - } - ], - "auditStamp": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "glossaryTerms", + "aspect": { + "json": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CustomerAccount" } + ], + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", - "changeType": "UPSERT", - "aspectName": "globalTags", - "aspect": { - "json": { - "tags": [ - { - "tag": "urn:li:tag:Legacy" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "json": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "TECHNICAL_OWNER" - }, - { - "owner": "urn:li:corpuser:jdoe", - "type": "TECHNICAL_OWNER" - } - ], - "lastModified": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:Legacy" } - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", - "changeType": "UPSERT", - "aspectName": "domains", - "aspect": { - "json": { - "domains": [ - "urn:li:domain:Engineering" - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", - "changeType": "UPSERT", - "aspectName": "editableDatasetProperties", - "aspect": { - "json": { - "created": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" - }, - "lastModified": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "TECHNICAL_OWNER" }, - "description": "new description" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "container", - "entityUrn": "urn:li:container:DATABASE", - "changeType": "UPSERT", - "aspectName": "glossaryTerms", - "aspect": { - "json": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CustomerAccount" - } - ], - "auditStamp": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + { + "owner": "urn:li:corpuser:jdoe", + "type": "TECHNICAL_OWNER" } + ], + "lastModified": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "container", - "entityUrn": "urn:li:container:DATABASE", - "changeType": "UPSERT", - "aspectName": "globalTags", - "aspect": { - "json": { - "tags": [ - { - "tag": "urn:li:tag:Legacy" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "container", - "entityUrn": "urn:li:container:DATABASE", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "json": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "TECHNICAL_OWNER" - }, - { - "owner": "urn:li:corpuser:jdoe", - "type": "TECHNICAL_OWNER" - } - ], - "lastModified": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "domains", + "aspect": { + "json": { + "domains": [ + "urn:li:domain:Engineering" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "editableDatasetProperties", + "aspect": { + "json": { + "created": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" + }, + "lastModified": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" + }, + "description": "new description" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:DATABASE", + "changeType": "UPSERT", + "aspectName": "glossaryTerms", + "aspect": { + "json": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CustomerAccount" } + ], + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "chart", - "entityUrn": "urn:li:chart:(looker,baz1)", - "changeType": "UPSERT", - "aspectName": "glossaryTerms", - "aspect": { - "json": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CustomerAccount" - } - ], - "auditStamp": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:DATABASE", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:Legacy" } - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "chart", - "entityUrn": "urn:li:chart:(looker,baz1)", - "changeType": "UPSERT", - "aspectName": "globalTags", - "aspect": { - "json": { - "tags": [ - { - "tag": "urn:li:tag:Legacy" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "chart", - "entityUrn": "urn:li:chart:(looker,baz1)", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "json": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "TECHNICAL_OWNER" - }, - { - "owner": "urn:li:corpuser:jdoe", - "type": "TECHNICAL_OWNER" - } - ], - "lastModified": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:DATABASE", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "TECHNICAL_OWNER" + }, + { + "owner": "urn:li:corpuser:jdoe", + "type": "TECHNICAL_OWNER" } + ], + "lastModified": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(looker,baz)", - "changeType": "UPSERT", - "aspectName": "glossaryTerms", - "aspect": { - "json": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CustomerAccount" - } - ], - "auditStamp": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:DATABASE", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,baz1)", + "changeType": "UPSERT", + "aspectName": "glossaryTerms", + "aspect": { + "json": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CustomerAccount" } + ], + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(looker,baz)", - "changeType": "UPSERT", - "aspectName": "globalTags", - "aspect": { - "json": { - "tags": [ - { - "tag": "urn:li:tag:Legacy" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(looker,baz)", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "json": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "TECHNICAL_OWNER" - }, - { - "owner": "urn:li:corpuser:jdoe", - "type": "TECHNICAL_OWNER" - } - ], - "lastModified": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,baz1)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:Legacy" } - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "mlFeature", - "entityUrn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BOOL_LIST_feature)", - "changeType": "UPSERT", - "aspectName": "glossaryTerms", - "aspect": { - "json": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CustomerAccount" - } - ], - "auditStamp": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,baz1)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "TECHNICAL_OWNER" + }, + { + "owner": "urn:li:corpuser:jdoe", + "type": "TECHNICAL_OWNER" } + ], + "lastModified": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "mlFeature", - "entityUrn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BOOL_LIST_feature)", - "changeType": "UPSERT", - "aspectName": "globalTags", - "aspect": { - "json": { - "tags": [ - { - "tag": "urn:li:tag:Legacy" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "mlFeature", - "entityUrn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BOOL_LIST_feature)", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "json": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "TECHNICAL_OWNER" - }, - { - "owner": "urn:li:corpuser:jdoe", - "type": "TECHNICAL_OWNER" - } - ], - "lastModified": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(looker,baz)", + "changeType": "UPSERT", + "aspectName": "glossaryTerms", + "aspect": { + "json": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CustomerAccount" } + ], + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "mlFeatureTable", - "entityUrn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,user_features)", - "changeType": "UPSERT", - "aspectName": "glossaryTerms", - "aspect": { - "json": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CustomerAccount" - } - ], - "auditStamp": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(looker,baz)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:Legacy" } - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "mlFeatureTable", - "entityUrn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,user_features)", - "changeType": "UPSERT", - "aspectName": "globalTags", - "aspect": { - "json": { - "tags": [ - { - "tag": "urn:li:tag:Legacy" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "mlFeatureTable", - "entityUrn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,user_features)", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "json": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "TECHNICAL_OWNER" - }, - { - "owner": "urn:li:corpuser:jdoe", - "type": "TECHNICAL_OWNER" - } - ], - "lastModified": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(looker,baz)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "TECHNICAL_OWNER" + }, + { + "owner": "urn:li:corpuser:jdoe", + "type": "TECHNICAL_OWNER" } + ], + "lastModified": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "mlPrimaryKey", - "entityUrn": "urn:li:mlPrimaryKey:(test_feature_table_all_feature_dtypes,dummy_entity_1)", - "changeType": "UPSERT", - "aspectName": "glossaryTerms", - "aspect": { - "json": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CustomerAccount" - } - ], - "auditStamp": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "mlFeature", + "entityUrn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BOOL_LIST_feature)", + "changeType": "UPSERT", + "aspectName": "glossaryTerms", + "aspect": { + "json": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CustomerAccount" } + ], + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "mlPrimaryKey", - "entityUrn": "urn:li:mlPrimaryKey:(test_feature_table_all_feature_dtypes,dummy_entity_1)", - "changeType": "UPSERT", - "aspectName": "globalTags", - "aspect": { - "json": { - "tags": [ - { - "tag": "urn:li:tag:Legacy" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "mlPrimaryKey", - "entityUrn": "urn:li:mlPrimaryKey:(test_feature_table_all_feature_dtypes,dummy_entity_1)", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "json": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "TECHNICAL_OWNER" - }, - { - "owner": "urn:li:corpuser:jdoe", - "type": "TECHNICAL_OWNER" - } - ], - "lastModified": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "mlFeature", + "entityUrn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BOOL_LIST_feature)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:Legacy" } - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "mlModel", - "entityUrn": "urn:li:mlModel:(urn:li:dataPlatform:science,scienceModel,PROD)", - "changeType": "UPSERT", - "aspectName": "glossaryTerms", - "aspect": { - "json": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CustomerAccount" - } - ], - "auditStamp": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "mlFeature", + "entityUrn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BOOL_LIST_feature)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "TECHNICAL_OWNER" + }, + { + "owner": "urn:li:corpuser:jdoe", + "type": "TECHNICAL_OWNER" } + ], + "lastModified": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "mlModel", - "entityUrn": "urn:li:mlModel:(urn:li:dataPlatform:science,scienceModel,PROD)", - "changeType": "UPSERT", - "aspectName": "globalTags", - "aspect": { - "json": { - "tags": [ - { - "tag": "urn:li:tag:Legacy" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "mlModel", - "entityUrn": "urn:li:mlModel:(urn:li:dataPlatform:science,scienceModel,PROD)", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "json": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "TECHNICAL_OWNER" - }, - { - "owner": "urn:li:corpuser:jdoe", - "type": "TECHNICAL_OWNER" - } - ], - "lastModified": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "mlFeatureTable", + "entityUrn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,user_features)", + "changeType": "UPSERT", + "aspectName": "glossaryTerms", + "aspect": { + "json": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CustomerAccount" } + ], + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "mlModelGroup", - "entityUrn": "urn:li:mlModelGroup:(urn:li:dataPlatform:science,scienceGroup,PROD)", - "changeType": "UPSERT", - "aspectName": "glossaryTerms", - "aspect": { - "json": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CustomerAccount" - } - ], - "auditStamp": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "mlFeatureTable", + "entityUrn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,user_features)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:Legacy" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "mlFeatureTable", + "entityUrn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,user_features)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "TECHNICAL_OWNER" + }, + { + "owner": "urn:li:corpuser:jdoe", + "type": "TECHNICAL_OWNER" } + ], + "lastModified": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "mlModelGroup", - "entityUrn": "urn:li:mlModelGroup:(urn:li:dataPlatform:science,scienceGroup,PROD)", - "changeType": "UPSERT", - "aspectName": "globalTags", - "aspect": { - "json": { - "tags": [ - { - "tag": "urn:li:tag:Legacy" - } - ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "mlPrimaryKey", + "entityUrn": "urn:li:mlPrimaryKey:(test_feature_table_all_feature_dtypes,dummy_entity_1)", + "changeType": "UPSERT", + "aspectName": "glossaryTerms", + "aspect": { + "json": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CustomerAccount" + } + ], + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "mlModelGroup", - "entityUrn": "urn:li:mlModelGroup:(urn:li:dataPlatform:science,scienceGroup,PROD)", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "json": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "TECHNICAL_OWNER" - }, - { - "owner": "urn:li:corpuser:jdoe", - "type": "TECHNICAL_OWNER" - } - ], - "lastModified": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "mlPrimaryKey", + "entityUrn": "urn:li:mlPrimaryKey:(test_feature_table_all_feature_dtypes,dummy_entity_1)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:Legacy" } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "mlPrimaryKey", + "entityUrn": "urn:li:mlPrimaryKey:(test_feature_table_all_feature_dtypes,dummy_entity_1)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "TECHNICAL_OWNER" + }, + { + "owner": "urn:li:corpuser:jdoe", + "type": "TECHNICAL_OWNER" + } + ], + "lastModified": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_123)", - "changeType": "UPSERT", - "aspectName": "glossaryTerms", - "aspect": { - "json": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CustomerAccount" - } - ], - "auditStamp": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "mlModel", + "entityUrn": "urn:li:mlModel:(urn:li:dataPlatform:science,scienceModel,PROD)", + "changeType": "UPSERT", + "aspectName": "glossaryTerms", + "aspect": { + "json": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CustomerAccount" } + ], + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_123)", - "changeType": "UPSERT", - "aspectName": "globalTags", - "aspect": { - "json": { - "tags": [ - { - "tag": "urn:li:tag:Legacy" - } - ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "mlModel", + "entityUrn": "urn:li:mlModel:(urn:li:dataPlatform:science,scienceModel,PROD)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:Legacy" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "mlModel", + "entityUrn": "urn:li:mlModel:(urn:li:dataPlatform:science,scienceModel,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "TECHNICAL_OWNER" + }, + { + "owner": "urn:li:corpuser:jdoe", + "type": "TECHNICAL_OWNER" + } + ], + "lastModified": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_123)", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "json": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "TECHNICAL_OWNER" - }, - { - "owner": "urn:li:corpuser:jdoe", - "type": "TECHNICAL_OWNER" - } - ], - "lastModified": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "mlModelGroup", + "entityUrn": "urn:li:mlModelGroup:(urn:li:dataPlatform:science,scienceGroup,PROD)", + "changeType": "UPSERT", + "aspectName": "glossaryTerms", + "aspect": { + "json": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CustomerAccount" } + ], + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(airflow,dag_abc,PROD)", - "changeType": "UPSERT", - "aspectName": "glossaryTerms", - "aspect": { - "json": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CustomerAccount" - } - ], - "auditStamp": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "mlModelGroup", + "entityUrn": "urn:li:mlModelGroup:(urn:li:dataPlatform:science,scienceGroup,PROD)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:Legacy" } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "mlModelGroup", + "entityUrn": "urn:li:mlModelGroup:(urn:li:dataPlatform:science,scienceGroup,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "TECHNICAL_OWNER" + }, + { + "owner": "urn:li:corpuser:jdoe", + "type": "TECHNICAL_OWNER" + } + ], + "lastModified": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(airflow,dag_abc,PROD)", - "changeType": "UPSERT", - "aspectName": "globalTags", - "aspect": { - "json": { - "tags": [ - { - "tag": "urn:li:tag:Legacy" - } - ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_123)", + "changeType": "UPSERT", + "aspectName": "glossaryTerms", + "aspect": { + "json": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CustomerAccount" + } + ], + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(airflow,dag_abc,PROD)", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "json": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "TECHNICAL_OWNER" - }, - { - "owner": "urn:li:corpuser:jdoe", - "type": "TECHNICAL_OWNER" - } - ], - "lastModified": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_123)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:Legacy" } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_123)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "TECHNICAL_OWNER" + }, + { + "owner": "urn:li:corpuser:jdoe", + "type": "TECHNICAL_OWNER" + } + ], + "lastModified": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "notebook", - "entityUrn": "urn:li:notebook:(querybook,1234)", - "changeType": "UPSERT", - "aspectName": "glossaryTerms", - "aspect": { - "json": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:CustomerAccount" - } - ], - "auditStamp": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,dag_abc,PROD)", + "changeType": "UPSERT", + "aspectName": "glossaryTerms", + "aspect": { + "json": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CustomerAccount" } + ], + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "notebook", - "entityUrn": "urn:li:notebook:(querybook,1234)", - "changeType": "UPSERT", - "aspectName": "globalTags", - "aspect": { - "json": { - "tags": [ - { - "tag": "urn:li:tag:Legacy" - } - ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,dag_abc,PROD)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:Legacy" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,dag_abc,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "TECHNICAL_OWNER" + }, + { + "owner": "urn:li:corpuser:jdoe", + "type": "TECHNICAL_OWNER" + } + ], + "lastModified": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "notebook", - "entityUrn": "urn:li:notebook:(querybook,1234)", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "json": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "TECHNICAL_OWNER" - }, - { - "owner": "urn:li:corpuser:jdoe", - "type": "TECHNICAL_OWNER" - } - ], - "lastModified": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "notebook", + "entityUrn": "urn:li:notebook:(querybook,1234)", + "changeType": "UPSERT", + "aspectName": "glossaryTerms", + "aspect": { + "json": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:CustomerAccount" } + ], + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" - } - }, - { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", - "changeType": "UPSERT", - "aspectName": "editableSchemaMetadata", - "aspect": { - "json": { - "created": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "notebook", + "entityUrn": "urn:li:notebook:(querybook,1234)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:Legacy" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "notebook", + "entityUrn": "urn:li:notebook:(querybook,1234)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "TECHNICAL_OWNER" }, - "editableSchemaFieldInfo": [ - { - "fieldPath": "field_foo", - "description": "field_foo!", - "glossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:AccountBalance" - } - ], - "auditStamp": { - "time": 1643871600000, - "actor": "urn:li:corpuser:ingestion" + { + "owner": "urn:li:corpuser:jdoe", + "type": "TECHNICAL_OWNER" + } + ], + "lastModified": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "editableSchemaMetadata", + "aspect": { + "json": { + "created": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "editableSchemaFieldInfo": [ + { + "fieldPath": "field_foo", + "description": "field_foo!", + "glossaryTerms": { + "terms": [ + { + "urn": "urn:li:glossaryTerm:AccountBalance" } - } - }, - { - "fieldPath": "field_bar", - "description": "field_bar?", - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Legacy" - } - ] + ], + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:ingestion" } } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "test-csv-enricher" + }, + { + "fieldPath": "field_bar", + "description": "field_bar?", + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Legacy" + } + ] + } + } + ] } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,baz1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,dag_abc,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(looker,baz)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:DATABASE", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Legacy", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "Legacy" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_123)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "mlFeature", + "entityUrn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BOOL_LIST_feature)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "mlModel", + "entityUrn": "urn:li:mlModel:(urn:li:dataPlatform:science,scienceModel,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "mlPrimaryKey", + "entityUrn": "urn:li:mlPrimaryKey:(test_feature_table_all_feature_dtypes,dummy_entity_1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "mlFeatureTable", + "entityUrn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,user_features)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "mlModelGroup", + "entityUrn": "urn:li:mlModelGroup:(urn:li:dataPlatform:science,scienceGroup,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" + } +}, +{ + "entityType": "notebook", + "entityUrn": "urn:li:notebook:(querybook,1234)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher" } - ] \ No newline at end of file +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json b/metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json index 58b08ee844028..52e92d27549f0 100644 --- a/metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json +++ b/metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json @@ -135,7 +135,7 @@ } }, "systemMetadata": { - "lastObserved": 1615443388097, + "lastObserved": 1672531200000, "runId": "delta-lake-test" } }, @@ -155,7 +155,7 @@ } }, "systemMetadata": { - "lastObserved": 1615443388097, + "lastObserved": 1672531200000, "runId": "delta-lake-test" } }, @@ -170,7 +170,7 @@ } }, "systemMetadata": { - "lastObserved": 1615443388097, + "lastObserved": 1672531200000, "runId": "delta-lake-test" } }, @@ -185,7 +185,7 @@ } }, "systemMetadata": { - "lastObserved": 1615443388097, + "lastObserved": 1672531200000, "runId": "delta-lake-test" } }, @@ -202,7 +202,22 @@ } }, "systemMetadata": { - "lastObserved": 1615443388097, + "lastObserved": 1672531200000, + "runId": "delta-lake-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1672531200000, "runId": "delta-lake-test" } }, @@ -222,7 +237,7 @@ } }, "systemMetadata": { - "lastObserved": 1615443388097, + "lastObserved": 1672531200000, "runId": "delta-lake-test" } }, @@ -237,7 +252,7 @@ } }, "systemMetadata": { - "lastObserved": 1615443388097, + "lastObserved": 1672531200000, "runId": "delta-lake-test" } }, @@ -252,7 +267,7 @@ } }, "systemMetadata": { - "lastObserved": 1615443388097, + "lastObserved": 1672531200000, "runId": "delta-lake-test" } }, @@ -269,7 +284,7 @@ } }, "systemMetadata": { - "lastObserved": 1615443388097, + "lastObserved": 1672531200000, "runId": "delta-lake-test" } }, @@ -284,7 +299,27 @@ } }, "systemMetadata": { - "lastObserved": 1615443388097, + "lastObserved": 1672531200000, + "runId": "delta-lake-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:acebf8bcf966274632d3d2b710ef4947", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd", + "urn": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1672531200000, "runId": "delta-lake-test" } }, @@ -299,7 +334,7 @@ } }, "systemMetadata": { - "lastObserved": 1615443388097, + "lastObserved": 1672531200000, "runId": "delta-lake-test" } }, @@ -310,7 +345,7 @@ "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1615443388097, + "timestampMillis": 1672531200000, "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" @@ -326,7 +361,31 @@ } }, "systemMetadata": { - "lastObserved": 1615443388097, + "lastObserved": 1672531200000, + "runId": "delta-lake-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-test-bucket/delta_tables/sales,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd", + "urn": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd" + }, + { + "id": "urn:li:container:acebf8bcf966274632d3d2b710ef4947", + "urn": "urn:li:container:acebf8bcf966274632d3d2b710ef4947" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1672531200000, "runId": "delta-lake-test" } } diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json index 8790f87c679bb..4dcdf71ce0095 100644 --- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json @@ -166,6 +166,26 @@ "runId": "allow_table.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:189046201d696e7810132cfa64dad337", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", @@ -250,6 +270,30 @@ "runId": "allow_table.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", @@ -334,6 +378,34 @@ "runId": "allow_table.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", @@ -418,6 +490,38 @@ "runId": "allow_table.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:a282913be26fceff334523c2be119df1", @@ -502,6 +606,42 @@ "runId": "allow_table.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a282913be26fceff334523c2be119df1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", @@ -656,6 +796,46 @@ "runId": "allow_table.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:a282913be26fceff334523c2be119df1", + "urn": "urn:li:container:a282913be26fceff334523c2be119df1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -838,6 +1018,46 @@ "runId": "allow_table.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/sales,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:a282913be26fceff334523c2be119df1", + "urn": "urn:li:container:a282913be26fceff334523c2be119df1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1089,6 +1309,46 @@ "runId": "allow_table.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:a282913be26fceff334523c2be119df1", + "urn": "urn:li:container:a282913be26fceff334523c2be119df1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1271,6 +1531,46 @@ "runId": "allow_table.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3df8f6b0f3a70d42cf70612a2fe5e5ef", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:a282913be26fceff334523c2be119df1", + "urn": "urn:li:container:a282913be26fceff334523c2be119df1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", @@ -1424,5 +1724,49 @@ "lastObserved": 1615443388097, "runId": "allow_table.json" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:a282913be26fceff334523c2be119df1", + "urn": "urn:li:container:a282913be26fceff334523c2be119df1" + }, + { + "id": "urn:li:container:3df8f6b0f3a70d42cf70612a2fe5e5ef", + "urn": "urn:li:container:3df8f6b0f3a70d42cf70612a2fe5e5ef" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json index 34d5244a42990..901e4c1262d3f 100644 --- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json @@ -164,6 +164,21 @@ "runId": "inner_table.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:974a39dc631803eddedc699cc9bb9759", @@ -246,6 +261,26 @@ "runId": "inner_table.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", @@ -328,6 +363,30 @@ "runId": "inner_table.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", @@ -410,6 +469,34 @@ "runId": "inner_table.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", @@ -492,6 +579,38 @@ "runId": "inner_table.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", @@ -646,6 +765,42 @@ "runId": "inner_table.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + }, + { + "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -828,6 +983,42 @@ "runId": "inner_table.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/sales,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + }, + { + "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1079,6 +1270,42 @@ "runId": "inner_table.json" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + }, + { + "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -1259,6 +1486,42 @@ "runId": "inner_table.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:6bb6dc6de93177210067d00b45b481bb", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + }, + { + "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", @@ -1412,5 +1675,45 @@ "lastObserved": 1615443388097, "runId": "inner_table.json" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + }, + { + "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" + }, + { + "id": "urn:li:container:6bb6dc6de93177210067d00b45b481bb", + "urn": "urn:li:container:6bb6dc6de93177210067d00b45b481bb" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json index b2ed4316111d0..18474e819334e 100644 --- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json @@ -164,6 +164,21 @@ "runId": "relative_path.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:85267d161e1a2ffa647cec6c1188549f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "relative_path.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)", @@ -317,5 +332,25 @@ "lastObserved": 1615443388097, "runId": "relative_path.json" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:85267d161e1a2ffa647cec6c1188549f", + "urn": "urn:li:container:85267d161e1a2ffa647cec6c1188549f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "relative_path.json" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json index afdc3d9360a48..bb47a077e878b 100644 --- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json @@ -163,6 +163,21 @@ "runId": "single_table.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:974a39dc631803eddedc699cc9bb9759", @@ -245,6 +260,26 @@ "runId": "single_table.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", @@ -327,6 +362,30 @@ "runId": "single_table.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", @@ -409,6 +468,34 @@ "runId": "single_table.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", @@ -491,6 +578,38 @@ "runId": "single_table.json" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", @@ -533,5 +652,41 @@ "lastObserved": 1615443388097, "runId": "single_table.json" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + }, + { + "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/feast/feast_repository_mces_golden.json b/metadata-ingestion/tests/integration/feast/feast_repository_mces_golden.json index d69ecda7473ea..1b91925289845 100644 --- a/metadata-ingestion/tests/integration/feast/feast_repository_mces_golden.json +++ b/metadata-ingestion/tests/integration/feast/feast_repository_mces_golden.json @@ -1,6 +1,5 @@ [ { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { "urn": "urn:li:mlPrimaryKey:(feature_store.driver_hourly_stats,driver_id)", @@ -14,7 +13,6 @@ "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { "description": "Driver ID", "dataType": "ORDINAL", - "version": null, "sources": [ "urn:li:dataset:(urn:li:dataPlatform:file,data.driver_stats_with_string.parquet,PROD)" ] @@ -23,17 +21,12 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "feast-repository-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "feast-repository-test" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { "urn": "urn:li:mlFeature:(feature_store.driver_hourly_stats,conv_rate)", @@ -47,7 +40,6 @@ "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { "description": "Conv rate", "dataType": "CONTINUOUS", - "version": null, "sources": [ "urn:li:dataset:(urn:li:dataPlatform:file,data.driver_stats_with_string.parquet,PROD)" ] @@ -56,17 +48,12 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "feast-repository-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "feast-repository-test" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { "urn": "urn:li:mlFeature:(feature_store.driver_hourly_stats,acc_rate)", @@ -80,7 +67,6 @@ "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { "description": "Acc rate", "dataType": "CONTINUOUS", - "version": null, "sources": [ "urn:li:dataset:(urn:li:dataPlatform:file,data.driver_stats_with_string.parquet,PROD)" ] @@ -89,17 +75,12 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "feast-repository-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "feast-repository-test" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { "urn": "urn:li:mlFeature:(feature_store.driver_hourly_stats,avg_daily_trips)", @@ -113,7 +94,6 @@ "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { "description": "Avg daily trips", "dataType": "ORDINAL", - "version": null, "sources": [ "urn:li:dataset:(urn:li:dataPlatform:file,data.driver_stats_with_string.parquet,PROD)" ] @@ -122,17 +102,12 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "feast-repository-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "feast-repository-test" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { "urn": "urn:li:mlFeature:(feature_store.driver_hourly_stats,string_feature)", @@ -146,7 +121,6 @@ "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { "description": "String feature", "dataType": "TEXT", - "version": null, "sources": [ "urn:li:dataset:(urn:li:dataPlatform:file,data.driver_stats_with_string.parquet,PROD)" ] @@ -155,17 +129,12 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "feast-repository-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "feast-repository-test" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,feature_store.driver_hourly_stats)", @@ -185,7 +154,6 @@ { "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { "customProperties": {}, - "description": null, "mlFeatures": [ "urn:li:mlFeature:(feature_store.driver_hourly_stats,conv_rate)", "urn:li:mlFeature:(feature_store.driver_hourly_stats,acc_rate)", @@ -200,17 +168,31 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "feast-repository-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "feast-repository-test" + } +}, +{ + "entityType": "mlFeatureTable", + "entityUrn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,feature_store.driver_hourly_stats)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "feature_store" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "feast-repository-test" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { "urn": "urn:li:mlFeature:(feature_store.transformed_conv_rate,conv_rate_plus_val1)", @@ -222,9 +204,7 @@ }, { "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, "dataType": "CONTINUOUS", - "version": null, "sources": [ "urn:li:dataset:(urn:li:dataPlatform:request,vals_to_add,PROD)", "urn:li:dataset:(urn:li:dataPlatform:file,data.driver_stats_with_string.parquet,PROD)" @@ -234,17 +214,12 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "feast-repository-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "feast-repository-test" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { "urn": "urn:li:mlFeature:(feature_store.transformed_conv_rate,conv_rate_plus_val2)", @@ -256,9 +231,7 @@ }, { "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, "dataType": "CONTINUOUS", - "version": null, "sources": [ "urn:li:dataset:(urn:li:dataPlatform:request,vals_to_add,PROD)", "urn:li:dataset:(urn:li:dataPlatform:file,data.driver_stats_with_string.parquet,PROD)" @@ -268,17 +241,12 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "feast-repository-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "feast-repository-test" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,feature_store.transformed_conv_rate)", @@ -298,7 +266,6 @@ { "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { "customProperties": {}, - "description": null, "mlFeatures": [ "urn:li:mlFeature:(feature_store.transformed_conv_rate,conv_rate_plus_val1)", "urn:li:mlFeature:(feature_store.transformed_conv_rate,conv_rate_plus_val2)" @@ -309,13 +276,28 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "feast-repository-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "feast-repository-test" + } +}, +{ + "entityType": "mlFeatureTable", + "entityUrn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,feature_store.transformed_conv_rate)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "feature_store" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "feast-repository-test" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/metabase/metabase_mces_golden.json b/metadata-ingestion/tests/integration/metabase/metabase_mces_golden.json index 3f307925ee9d5..6e57dfaae0ce0 100644 --- a/metadata-ingestion/tests/integration/metabase/metabase_mces_golden.json +++ b/metadata-ingestion/tests/integration/metabase/metabase_mces_golden.json @@ -1,30 +1,41 @@ [ { "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { - "urn": "urn:li:dashboard:(metabase,1)", + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(metabase,1)", "aspects": [ { - "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": {}, - "title": "Dashboard 1", + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Metrics": "", + "Filters": "", + "Dimensions": "customer_id, first_name, last_name, amount, payment_date, rental_id" + }, + "title": "Customer Payment", "description": "", - "charts": [ - "urn:li:chart:(metabase,1)", - "urn:li:chart:(metabase,2)" - ], - "datasets": [], "lastModified": { "created": { - "time": 1639417721742, + "time": 1639417592792, "actor": "urn:li:corpuser:admin@metabase.com" }, "lastModified": { - "time": 1639417721742, + "time": 1639417592792, "actor": "urn:li:corpuser:admin@metabase.com" } }, - "dashboardUrl": "http://localhost:3000/dashboard/1" + "chartUrl": "http://localhost:3000/card/1", + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:bigquery,acryl-data.public.payment,PROD)" + } + ], + "type": "TABLE" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartQuery": { + "rawQuery": "SELECT\\n\\tcustomer.customer_id,\\n\\tfirst_name,\\n\\tlast_name,\\n\\tamount,\\n\\tpayment_date,\\n\\trental_id\\nFROM\\n\\tcustomer\\nINNER JOIN payment \\n ON payment.customer_id = customer.customer_id\\nORDER BY payment_date", + "type": "SQL" } }, { @@ -52,40 +63,34 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(metabase,1)", + "urn": "urn:li:chart:(metabase,2)", "aspects": [ { "com.linkedin.pegasus2avro.chart.ChartInfo": { "customProperties": { - "Metrics": "", - "Filters": "", - "Dimensions": "customer_id, first_name, last_name, amount, payment_date, rental_id" + "Metrics": "Count", + "Filters": "['=', ['field', 136, None], 2006]", + "Dimensions": "Rating" }, - "title": "Customer Payment", + "title": "Films, Count, Grouped by Rating, Filtered by Release Year, Sorted by [Unknown Field] descending", "description": "", "lastModified": { "created": { - "time": 1639417592792, + "time": 1636614000000, "actor": "urn:li:corpuser:admin@metabase.com" }, "lastModified": { - "time": 1639417592792, + "time": 1636614000000, "actor": "urn:li:corpuser:admin@metabase.com" } }, - "chartUrl": "http://localhost:3000/card/1", + "chartUrl": "http://localhost:3000/card/2", "inputs": [ { - "string": "urn:li:dataset:(urn:li:dataPlatform:bigquery,acryl-data.public.payment,PROD)" + "string": "urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.film,PROD)" } ], - "type": "TABLE" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartQuery": { - "rawQuery": "SELECT\\n\\tcustomer.customer_id,\\n\\tfirst_name,\\n\\tlast_name,\\n\\tamount,\\n\\tpayment_date,\\n\\trental_id\\nFROM\\n\\tcustomer\\nINNER JOIN payment \\n ON payment.customer_id = customer.customer_id\\nORDER BY payment_date", - "type": "SQL" + "type": "BAR" } }, { @@ -112,36 +117,30 @@ }, { "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(metabase,2)", + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(metabase,1)", "aspects": [ { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Metrics": "Count", - "Filters": "['=', ['field', 136, None], 2006]", - "Dimensions": "Rating" - }, - "title": "Films, Count, Grouped by Rating, Filtered by Release Year, Sorted by [Unknown Field] descending", + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "Dashboard 1", "description": "", + "charts": [ + "urn:li:chart:(metabase,1)", + "urn:li:chart:(metabase,2)" + ], + "datasets": [], "lastModified": { "created": { - "time": 1639417717110, + "time": 1639417721742, "actor": "urn:li:corpuser:admin@metabase.com" }, "lastModified": { - "time": 1639417717110, + "time": 1639417721742, "actor": "urn:li:corpuser:admin@metabase.com" - }, - "deleted": null - }, - "chartUrl": "http://localhost:3000/card/2", - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.film,PROD)" } - ], - "type": "BAR" + }, + "dashboardUrl": "http://localhost:3000/dashboard/1" } }, { @@ -165,5 +164,50 @@ "lastObserved": 1636614000000, "runId": "metabase-test" } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(metabase,1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1636614000000, + "runId": "metabase-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(metabase,2)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1636614000000, + "runId": "metabase-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(metabase,1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1636614000000, + "runId": "metabase-test" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/mode/mode_mces_golden.json b/metadata-ingestion/tests/integration/mode/mode_mces_golden.json index 003a74ed0a6d1..62e14520e53e8 100644 --- a/metadata-ingestion/tests/integration/mode/mode_mces_golden.json +++ b/metadata-ingestion/tests/integration/mode/mode_mces_golden.json @@ -1,6 +1,5 @@ [ { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { "urn": "urn:li:dashboard:(mode,2934237)", @@ -8,7 +7,6 @@ { "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { "customProperties": {}, - "externalUrl": null, "title": "Report 1", "description": "First Report", "charts": [ @@ -18,19 +16,14 @@ "lastModified": { "created": { "time": 1639169724316, - "actor": "urn:li:corpuser:modeuser", - "impersonator": null + "actor": "urn:li:corpuser:modeuser" }, "lastModified": { "time": 1639182684451, - "actor": "urn:li:corpuser:modeuser", - "impersonator": null - }, - "deleted": null + "actor": "urn:li:corpuser:modeuser" + } }, - "dashboardUrl": "https://app.mode.com/acryl/reports/9d2da37fa91e", - "access": null, - "lastRefreshed": null + "dashboardUrl": "https://app.mode.com/acryl/reports/9d2da37fa91e" } }, { @@ -45,31 +38,49 @@ "owners": [ { "owner": "urn:li:corpuser:modeuser", - "type": "DATAOWNER", - "source": null + "type": "DATAOWNER" } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1638860400000, - "runId": "mode-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "mode-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(mode,2934237)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "acryl" + }, + { + "id": "AcrylTest" + }, + { + "id": "Report 1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "mode-test" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { "urn": "urn:li:chart:(mode,f622b9ee725b)", @@ -80,37 +91,31 @@ "Columns": "payment_date,Staff First Name,amount,Staff Last Name", "Filters": "amount" }, - "externalUrl": null, "title": "Customer Staff Table", "description": "", "lastModified": { "created": { "time": 1639170083088, - "actor": "urn:li:corpuser:modeuser", - "impersonator": null + "actor": "urn:li:corpuser:modeuser" }, "lastModified": { "time": 1639182684438, - "actor": "urn:li:corpuser:modeuser", - "impersonator": null - }, - "deleted": null + "actor": "urn:li:corpuser:modeuser" + } }, "chartUrl": "https://app.mode.com/acryltest/reports/9d2da37fa91e/viz/f622b9ee725b", "inputs": [ { - "string": "urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.rental,PROD)" + "string": "urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.staff,PROD)" }, { "string": "urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.payment,PROD)" }, { - "string": "urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.staff,PROD)" + "string": "urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.rental,PROD)" } ], - "type": "TABLE", - "access": null, - "lastRefreshed": null + "type": "TABLE" } }, { @@ -131,27 +136,82 @@ "owners": [ { "owner": "urn:li:corpuser:modeuser", - "type": "DATAOWNER", - "source": null + "type": "DATAOWNER" } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1638860400000, - "runId": "mode-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "mode-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(mode,f622b9ee725b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "mode-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(mode,f622b9ee725b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "acryl" + }, + { + "id": "AcrylTest" + }, + { + "id": "Report 1" + }, + { + "id": "Customer and staff" + }, + { + "id": "Customer Staff Table" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "mode-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(mode,2934237)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "mode-test" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json b/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json index 7d69ee10196f6..1f662cfe514e2 100644 --- a/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json +++ b/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json @@ -4132,5 +4132,65 @@ "lastObserved": 1615443388097, "runId": "mongodb-test" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,mngdb.emptyCollection,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mongodb-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,mngdb.firstCollection,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mongodb-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,mngdb.largeCollection,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mongodb-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mongodb,mngdb.secondCollection,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mongodb-test" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_cluster.json b/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_cluster.json index c38b134a97243..e0ab6bbec7b30 100644 --- a/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_cluster.json +++ b/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_cluster.json @@ -652,5 +652,215 @@ "lastObserved": 1638532800000, "runId": "nifi-test-cluster" } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638532800000, + "runId": "nifi-test-cluster" + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),3ec2acd6-a0d4-3198-9066-a59fb757bc05)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638532800000, + "runId": "nifi-test-cluster" + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),8eb5263d-017d-1000-ffff-ffff911b23aa)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638532800000, + "runId": "nifi-test-cluster" + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),8a218b6e-e6a0-36b6-bc4b-79d202a80167)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638532800000, + "runId": "nifi-test-cluster" + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),71bc17ed-a3bc-339a-a100-ebad434717d4)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638532800000, + "runId": "nifi-test-cluster" + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),c5f6fc66-ffbb-3f60-9564-f2466ae32493)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638532800000, + "runId": "nifi-test-cluster" + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),c8c73d4c-ebdd-1bee-9b46-629672cd11a0)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638532800000, + "runId": "nifi-test-cluster" + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),8eb55aeb-017d-1000-ffff-fffff475768d)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638532800000, + "runId": "nifi-test-cluster" + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,80820b2f-017d-1000-85cf-05f56cde9185,PROD),fed5914b-937b-37dd-89c0-b34ffbae9cf4)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638532800000, + "runId": "nifi-test-cluster" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638532800000, + "runId": "nifi-test-cluster" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:nifi,default.s3_data,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638532800000, + "runId": "nifi-test-cluster" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host.temperature,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638532800000, + "runId": "nifi-test-cluster" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:nifi,default.sftp_files_out,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638532800000, + "runId": "nifi-test-cluster" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638532800000, + "runId": "nifi-test-cluster" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_standalone.json b/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_standalone.json index 5fe65a651ee65..48a22bebb3f8e 100644 --- a/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_standalone.json +++ b/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_standalone.json @@ -227,5 +227,80 @@ "lastObserved": 1638532800000, "runId": "nifi-test-standalone" } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638532800000, + "runId": "nifi-test-standalone" + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),91d59f03-1c2b-3f3f-48bc-f89296a328bd)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638532800000, + "runId": "nifi-test-standalone" + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),aed63edf-e660-3f29-b56b-192cf6286889)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638532800000, + "runId": "nifi-test-standalone" + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),cb7693ed-f93b-3340-3776-fe80e6283ddc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638532800000, + "runId": "nifi-test-standalone" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638532800000, + "runId": "nifi-test-standalone" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/openapi/openapi_mces_golden.json b/metadata-ingestion/tests/integration/openapi/openapi_mces_golden.json index ede0006ef08d7..ad270857dd7fc 100755 --- a/metadata-ingestion/tests/integration/openapi/openapi_mces_golden.json +++ b/metadata-ingestion/tests/integration/openapi/openapi_mces_golden.json @@ -148,5 +148,35 @@ "lastObserved": 1586847600000, "runId": "openapi-2020_04_14-07_00_00" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:OpenApi,test_openapi.root,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "openapi-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:OpenApi,test_openapi.v2,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "openapi-2020_04_14-07_00_00" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json index 06decfe7d5b4a..fb1c42705c8e8 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json @@ -778,6 +778,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)" + }, { "id": "demo-workspace" } @@ -884,6 +888,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)" + }, { "id": "demo-workspace" } @@ -1016,6 +1024,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:powerbi,aws-ap-south-1)" + }, { "id": "demo-workspace" } diff --git a/metadata-ingestion/tests/integration/powerbi_report_server/golden_test_fail_api_ingest.json b/metadata-ingestion/tests/integration/powerbi_report_server/golden_test_fail_api_ingest.json index 69a567654cac1..43a7080b17392 100644 --- a/metadata-ingestion/tests/integration/powerbi_report_server/golden_test_fail_api_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi_report_server/golden_test_fail_api_ingest.json @@ -161,6 +161,40 @@ "runId": "powerbi-report-server-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "powerbi_report_server" + }, + { + "id": "server_alias" + }, + { + "id": "Reports" + }, + { + "id": "path" + }, + { + "id": "to" + }, + { + "id": "Testa" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-report-server-test" + } +}, { "entityType": "corpuser", "entityUrn": "urn:li:corpuser:TEST_USER", @@ -322,5 +356,39 @@ "lastObserved": 1643871600000, "runId": "powerbi-report-server-test" } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938d)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "powerbi_report_server" + }, + { + "id": "server_alias" + }, + { + "id": "Reports" + }, + { + "id": "path" + }, + { + "id": "to" + }, + { + "id": "Testd" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-report-server-test" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi_report_server/golden_test_ingest.json b/metadata-ingestion/tests/integration/powerbi_report_server/golden_test_ingest.json index f4277e41d58c7..6c36f7b0e33e4 100644 --- a/metadata-ingestion/tests/integration/powerbi_report_server/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi_report_server/golden_test_ingest.json @@ -161,6 +161,40 @@ "runId": "powerbi-report-server-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "powerbi_report_server" + }, + { + "id": "server_alias" + }, + { + "id": "Reports" + }, + { + "id": "path" + }, + { + "id": "to" + }, + { + "id": "Testa" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-report-server-test" + } +}, { "entityType": "corpuser", "entityUrn": "urn:li:corpuser:TEST_USER", @@ -312,6 +346,40 @@ "runId": "powerbi-report-server-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938c)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "powerbi_report_server" + }, + { + "id": "server_alias" + }, + { + "id": "Reports" + }, + { + "id": "path" + }, + { + "id": "to" + }, + { + "id": "Testc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-report-server-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938c)", @@ -505,6 +573,40 @@ "runId": "powerbi-report-server-test" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938d)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "powerbi_report_server" + }, + { + "id": "server_alias" + }, + { + "id": "Reports" + }, + { + "id": "path" + }, + { + "id": "to" + }, + { + "id": "Testd" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-report-server-test" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,reports.ee56dc21-248a-4138-a446-ee5ab1fc938d)", diff --git a/metadata-ingestion/tests/integration/remote/golden/remote_enricher_golden.json b/metadata-ingestion/tests/integration/remote/golden/remote_enricher_golden.json index 3a5970c1ce70b..b3a86bbd8616b 100644 --- a/metadata-ingestion/tests/integration/remote/golden/remote_enricher_golden.json +++ b/metadata-ingestion/tests/integration/remote/golden/remote_enricher_golden.json @@ -158,5 +158,35 @@ "lastObserved": 1629795600000, "runId": "remote-1" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1629795600000, + "runId": "remote-1" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Legacy", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "Legacy" + } + }, + "systemMetadata": { + "lastObserved": 1629795600000, + "runId": "remote-1" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/remote/golden/remote_lineage_golden.json b/metadata-ingestion/tests/integration/remote/golden/remote_lineage_golden.json index 085e2e7b33a7e..4297be4eac62f 100644 --- a/metadata-ingestion/tests/integration/remote/golden/remote_lineage_golden.json +++ b/metadata-ingestion/tests/integration/remote/golden/remote_lineage_golden.json @@ -6,7 +6,6 @@ "aspectName": "upstreamLineage", "aspect": { "json": { - "fineGrainedLineages": [], "upstreams": [ { "auditStamp": { @@ -24,7 +23,8 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:kafka,topic1,DEV)", "type": "TRANSFORMED" } - ] + ], + "fineGrainedLineages": [] } }, "systemMetadata": { @@ -39,7 +39,6 @@ "aspectName": "upstreamLineage", "aspect": { "json": { - "fineGrainedLineages": [], "upstreams": [ { "auditStamp": { @@ -49,7 +48,38 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test.kafka.topic2,PROD)", "type": "TRANSFORMED" } - ] + ], + "fineGrainedLineages": [] + } + }, + "systemMetadata": { + "lastObserved": 1629795600000, + "runId": "remote-3" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,topic2,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1629795600000, + "runId": "remote-3" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,topic3,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false } }, "systemMetadata": { @@ -57,4 +87,4 @@ "runId": "remote-3" } } -] +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_file_without_extension.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_file_without_extension.json index 818a8b7a5adde..d042c3fbb158b 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_file_without_extension.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_file_without_extension.json @@ -211,7 +211,12 @@ "aspectName": "browsePathsV2", "aspect": { "json": { - "path": [] + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + } + ] } }, "systemMetadata": { @@ -311,6 +316,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" @@ -415,6 +424,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" @@ -523,6 +536,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" @@ -635,6 +652,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" @@ -751,6 +772,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" @@ -871,6 +896,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" @@ -995,6 +1024,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" @@ -1123,6 +1156,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" @@ -1201,6 +1238,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_files.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_files.json index 7fc7e79a643b7..8e4fcb80ff855 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_files.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_files.json @@ -523,7 +523,12 @@ "aspectName": "browsePathsV2", "aspect": { "json": { - "path": [] + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + } + ] } }, "systemMetadata": { @@ -623,6 +628,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" @@ -727,6 +736,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" @@ -835,6 +848,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" @@ -947,6 +964,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" @@ -1063,6 +1084,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" @@ -1183,6 +1208,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" @@ -1307,6 +1336,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" @@ -2663,6 +2696,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" @@ -3196,6 +3233,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" @@ -3745,6 +3786,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" @@ -3953,6 +3998,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" @@ -4456,6 +4505,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" @@ -4818,6 +4871,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" @@ -7489,6 +7546,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:file,test-platform-instance)" + }, { "id": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0", "urn": "urn:li:container:c0b6448a96b5b99a7cabec1c4bfa66c0" diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json index cb7ccf87f7c6f..58b81065c190f 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json @@ -211,7 +211,12 @@ "aspectName": "browsePathsV2", "aspect": { "json": { - "path": [] + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + } + ] } }, "systemMetadata": { @@ -311,6 +316,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + }, { "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" @@ -415,6 +424,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + }, { "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" @@ -523,6 +536,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + }, { "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" @@ -635,6 +652,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + }, { "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" @@ -697,6 +718,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + }, { "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json index 40e45150fa211..0c1d92ed58e3d 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json @@ -523,7 +523,12 @@ "aspectName": "browsePathsV2", "aspect": { "json": { - "path": [] + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + } + ] } }, "systemMetadata": { @@ -623,6 +628,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + }, { "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" @@ -727,6 +736,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + }, { "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" @@ -835,6 +848,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + }, { "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" @@ -878,6 +895,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + }, { "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" @@ -1061,6 +1082,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + }, { "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" @@ -1244,6 +1269,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + }, { "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" @@ -1403,6 +1432,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + }, { "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" @@ -1586,6 +1619,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + }, { "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" @@ -1769,6 +1806,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + }, { "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" @@ -2264,6 +2305,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + }, { "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" diff --git a/metadata-ingestion/tests/integration/salesforce/salesforce_mces_golden.json b/metadata-ingestion/tests/integration/salesforce/salesforce_mces_golden.json index 3b498707c4fb4..4e54f199eafed 100644 --- a/metadata-ingestion/tests/integration/salesforce/salesforce_mces_golden.json +++ b/metadata-ingestion/tests/integration/salesforce/salesforce_mces_golden.json @@ -1974,5 +1974,65 @@ "lastObserved": 1652353200000, "runId": "salesforce-test" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:salesforce,Account,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1652353200000, + "runId": "salesforce-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:salesforce,Property__c,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1652353200000, + "runId": "salesforce-test" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Custom", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "Custom" + } + }, + "systemMetadata": { + "lastObserved": 1652353200000, + "runId": "salesforce-test" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:SystemField", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "SystemField" + } + }, + "systemMetadata": { + "lastObserved": 1652353200000, + "runId": "salesforce-test" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/starburst-trino-usage/trino_usages_golden.json b/metadata-ingestion/tests/integration/starburst-trino-usage/trino_usages_golden.json index 55fa1ff9a2dc1..f6c919edc2f45 100644 --- a/metadata-ingestion/tests/integration/starburst-trino-usage/trino_usages_golden.json +++ b/metadata-ingestion/tests/integration/starburst-trino-usage/trino_usages_golden.json @@ -1,21 +1,62 @@ [ { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,testcatalog.testschema.testtable,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "datasetUsageStatistics", "aspect": { - "value": "{\"timestampMillis\": 1634169600000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"uniqueUserCount\": 1, \"totalSqlQueries\": 2, \"topSqlQueries\": [\"select * from testcatalog.testschema.testtable limit 100\"], \"userCounts\": [{\"user\": \"urn:li:corpuser:test-name\", \"count\": 2, \"userEmail\": \"test-name@acryl.io\"}], \"fieldCounts\": [{\"fieldPath\": \"column1\", \"count\": 2}, {\"fieldPath\": \"column2\", \"count\": 2}]}", - "contentType": "application/json" + "json": { + "timestampMillis": 1634169600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "uniqueUserCount": 1, + "totalSqlQueries": 2, + "topSqlQueries": [ + "select * from testcatalog.testschema.testtable limit 100" + ], + "userCounts": [ + { + "user": "urn:li:corpuser:test-name", + "count": 2, + "userEmail": "test-name@acryl.io" + } + ], + "fieldCounts": [ + { + "fieldPath": "column1", + "count": 2 + }, + { + "fieldPath": "column2", + "count": 2 + } + ] + } }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "test-trino-usage", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test-trino-usage" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,testcatalog.testschema.testtable,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1629795600000, + "runId": "test-trino-usage" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json index f82cf56ce2b81..65e74b1899069 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json @@ -74,7 +74,12 @@ "aspectName": "browsePathsV2", "aspect": { "json": { - "path": [] + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + } + ] } }, "systemMetadata": { @@ -157,7 +162,12 @@ "aspectName": "browsePathsV2", "aspect": { "json": { - "path": [] + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + } + ] } }, "systemMetadata": { @@ -282,6 +292,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -430,6 +444,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -559,6 +577,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -688,6 +710,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -980,6 +1006,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -1542,6 +1572,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -2156,6 +2190,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -2666,6 +2704,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -2812,6 +2854,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -3126,6 +3172,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -3324,6 +3374,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -3826,6 +3880,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -4296,6 +4354,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -4798,6 +4860,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -5181,6 +5247,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -5544,6 +5614,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -5930,6 +6004,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -6371,6 +6449,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -6786,6 +6868,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -7117,6 +7203,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -7480,6 +7570,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -7733,6 +7827,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -8174,6 +8272,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -8511,6 +8613,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -8874,6 +8980,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -9263,6 +9373,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -9704,6 +9818,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -9893,6 +10011,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -10019,6 +10141,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -10122,6 +10248,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -10223,6 +10353,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -10338,6 +10472,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -12530,6 +12668,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -12927,6 +13069,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -13436,6 +13582,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -14682,6 +14832,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -21503,6 +21657,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -25653,6 +25811,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -31027,6 +31189,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -31441,6 +31607,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -31882,6 +32052,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -32938,6 +33112,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "Samples" } @@ -33157,6 +33335,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "urn:li:container:66fa1e14620418276c85f3b552c7ec65", "urn": "urn:li:container:66fa1e14620418276c85f3b552c7ec65" @@ -33386,6 +33568,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "default" }, @@ -33581,6 +33767,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "default" }, @@ -33857,6 +34047,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "default" }, @@ -34107,6 +34301,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "default" }, @@ -34305,6 +34503,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "default" }, @@ -34490,6 +34692,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "default" }, @@ -34662,6 +34868,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "default" }, @@ -34782,6 +34992,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "default" }, @@ -34876,6 +35090,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "Samples" }, @@ -34967,6 +35185,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "Samples" }, @@ -35305,6 +35527,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "Samples" }, @@ -36178,6 +36404,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "default" }, @@ -37156,6 +37386,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "default" }, @@ -38212,6 +38446,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "default" }, @@ -39125,6 +39363,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "default" }, @@ -39479,6 +39721,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "default" }, @@ -40418,6 +40664,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "default" }, @@ -41513,6 +41763,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "default" }, @@ -42465,6 +42719,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "default" }, @@ -42511,6 +42769,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "default" }, @@ -42554,6 +42816,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "default" }, @@ -42596,6 +42862,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, { "id": "default" }, diff --git a/metadata-ingestion/tests/unit/sagemaker/sagemaker_mces_golden.json b/metadata-ingestion/tests/unit/sagemaker/sagemaker_mces_golden.json index ceda91359b8d0..8163545155302 100644 --- a/metadata-ingestion/tests/unit/sagemaker/sagemaker_mces_golden.json +++ b/metadata-ingestion/tests/unit/sagemaker/sagemaker_mces_golden.json @@ -1,1815 +1,2095 @@ [ - { - "auditHeader": null, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test-2,some-feature-1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "TEXT", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:s3,datahub-sagemaker-outputs,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:glue,sagemaker_featurestore.test-2-123412341234,PROD)" - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { - "urn": "urn:li:mlPrimaryKey:(test-2,some-feature-2)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { - "description": null, - "dataType": "ORDINAL", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:s3,datahub-sagemaker-outputs,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:glue,sagemaker_featurestore.test-2-123412341234,PROD)" - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test-2,some-feature-3)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "CONTINUOUS", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:s3,datahub-sagemaker-outputs,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:glue,sagemaker_featurestore.test-2-123412341234,PROD)" - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test-2,some-feature-1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "dataType": "TEXT", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:s3,datahub-sagemaker-outputs,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:glue,sagemaker_featurestore.test-2-123412341234,PROD)" + ] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { - "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:sagemaker,test-2)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/sagemaker" - ] - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { - "customProperties": { - "arn": "arn:aws:sagemaker:us-west-2:123412341234:feature-group/test-2", - "creation_time": "2021-06-24 09:48:37.035000", - "status": "Created" - }, - "description": "Yet another test feature group", - "mlFeatures": [ - "urn:li:mlFeature:(test-2,some-feature-1)", - "urn:li:mlFeature:(test-2,some-feature-3)" - ], - "mlPrimaryKeys": [ - "urn:li:mlPrimaryKey:(test-2,some-feature-2)" - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { + "urn": "urn:li:mlPrimaryKey:(test-2,some-feature-2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { + "dataType": "ORDINAL", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:s3,datahub-sagemaker-outputs,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:glue,sagemaker_featurestore.test-2-123412341234,PROD)" + ] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test-1,name)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "TEXT", - "version": null, - "sources": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test-2,some-feature-3)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "dataType": "CONTINUOUS", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:s3,datahub-sagemaker-outputs,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:glue,sagemaker_featurestore.test-2-123412341234,PROD)" + ] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { - "urn": "urn:li:mlPrimaryKey:(test-1,id)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { - "description": null, - "dataType": "ORDINAL", - "version": null, - "sources": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { + "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:sagemaker,test-2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/sagemaker" + ] + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { + "customProperties": { + "arn": "arn:aws:sagemaker:us-west-2:123412341234:feature-group/test-2", + "creation_time": "2021-06-24 09:48:37.035000", + "status": "Created" + }, + "description": "Yet another test feature group", + "mlFeatures": [ + "urn:li:mlFeature:(test-2,some-feature-1)", + "urn:li:mlFeature:(test-2,some-feature-3)" + ], + "mlPrimaryKeys": [ + "urn:li:mlPrimaryKey:(test-2,some-feature-2)" + ] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test-1,height)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "CONTINUOUS", - "version": null, - "sources": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test-1,name)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "dataType": "TEXT", + "sources": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test-1,time)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "TEXT", - "version": null, - "sources": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { + "urn": "urn:li:mlPrimaryKey:(test-1,id)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { + "dataType": "ORDINAL", + "sources": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { - "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:sagemaker,test-1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/sagemaker" - ] - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { - "customProperties": { - "arn": "arn:aws:sagemaker:us-west-2:123412341234:feature-group/test-1", - "creation_time": "2021-06-23 13:58:10.264000", - "status": "Created" - }, - "description": "First test feature group", - "mlFeatures": [ - "urn:li:mlFeature:(test-1,name)", - "urn:li:mlFeature:(test-1,height)", - "urn:li:mlFeature:(test-1,time)" - ], - "mlPrimaryKeys": [ - "urn:li:mlPrimaryKey:(test-1,id)" - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test-1,height)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "dataType": "CONTINUOUS", + "sources": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { - "urn": "urn:li:mlPrimaryKey:(test,feature_1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { - "description": null, - "dataType": "TEXT", - "version": null, - "sources": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test-1,time)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "dataType": "TEXT", + "sources": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test,feature_2)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "ORDINAL", - "version": null, - "sources": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { + "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:sagemaker,test-1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/sagemaker" + ] + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { + "customProperties": { + "arn": "arn:aws:sagemaker:us-west-2:123412341234:feature-group/test-1", + "creation_time": "2021-06-23 13:58:10.264000", + "status": "Created" + }, + "description": "First test feature group", + "mlFeatures": [ + "urn:li:mlFeature:(test-1,name)", + "urn:li:mlFeature:(test-1,height)", + "urn:li:mlFeature:(test-1,time)" + ], + "mlPrimaryKeys": [ + "urn:li:mlPrimaryKey:(test-1,id)" + ] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test,feature_3)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "CONTINUOUS", - "version": null, - "sources": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { + "urn": "urn:li:mlPrimaryKey:(test,feature_1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { + "dataType": "TEXT", + "sources": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { - "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:sagemaker,test)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/sagemaker" - ] - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { - "customProperties": { - "arn": "arn:aws:sagemaker:us-west-2:123412341234:feature-group/test", - "creation_time": "2021-06-14 11:03:00.803000", - "status": "Created" - }, - "description": null, - "mlFeatures": [ - "urn:li:mlFeature:(test,feature_2)", - "urn:li:mlFeature:(test,feature_3)" - ], - "mlPrimaryKeys": [ - "urn:li:mlPrimaryKey:(test,feature_1)" - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test,feature_2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "dataType": "ORDINAL", + "sources": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,auto-ml-job-input-bucket/file_txt,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "dataset_type": "s3", - "uri": "s3://auto-ml-job-input-bucket/file.txt", - "datatype": "ManifestFile" - }, - "externalUrl": null, - "description": null, - "uri": null, - "tags": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test,feature_3)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "dataType": "CONTINUOUS", + "sources": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,auto-ml-job-output-bucket/file_txt,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "dataset_type": "s3", - "uri": "s3://auto-ml-job-output-bucket/file.txt" - }, - "externalUrl": null, - "description": null, - "uri": null, - "tags": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { + "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:sagemaker,test)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/sagemaker" + ] + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { + "customProperties": { + "arn": "arn:aws:sagemaker:us-west-2:123412341234:feature-group/test", + "creation_time": "2021-06-14 11:03:00.803000", + "status": "Created" + }, + "mlFeatures": [ + "urn:li:mlFeature:(test,feature_2)", + "urn:li:mlFeature:(test,feature_3)" + ], + "mlPrimaryKeys": [ + "urn:li:mlPrimaryKey:(test,feature_1)" + ] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,compilation-job-bucket/input-config.tar_gz,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "dataset_type": "s3", - "uri": "s3://compilation-job-bucket/input-config.tar.gz", - "framework": "TENSORFLOW", - "framework_version": "string" - }, - "externalUrl": null, - "description": null, - "uri": null, - "tags": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,auto-ml-job-input-bucket/file_txt,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "dataset_type": "s3", + "uri": "s3://auto-ml-job-input-bucket/file.txt", + "datatype": "ManifestFile" + }, + "tags": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,compilation-job-bucket/output-config.tar_gz,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "dataset_type": "s3", - "uri": "s3://compilation-job-bucket/output-config.tar.gz", - "target_device": "lambda", - "target_platform": "{'Os': 'ANDROID', 'Arch': 'X86_64', 'Accelerator': 'INTEL_GRAPHICS'}" - }, - "externalUrl": null, - "description": null, - "uri": null, - "tags": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,auto-ml-job-output-bucket/file_txt,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "dataset_type": "s3", + "uri": "s3://auto-ml-job-output-bucket/file.txt" + }, + "tags": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,edge-packaging-bucket/model-artifact.tar_gz,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "dataset_type": "s3", - "uri": "s3://edge-packaging-bucket/model-artifact.tar.gz" - }, - "externalUrl": null, - "description": null, - "uri": null, - "tags": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,compilation-job-bucket/input-config.tar_gz,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "dataset_type": "s3", + "uri": "s3://compilation-job-bucket/input-config.tar.gz", + "framework": "TENSORFLOW", + "framework_version": "string" + }, + "tags": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,edge-packaging-bucket/output-config.tar_gz,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "dataset_type": "s3", - "uri": "s3://edge-packaging-bucket/output-config.tar.gz" - }, - "externalUrl": null, - "description": null, - "uri": null, - "tags": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,compilation-job-bucket/output-config.tar_gz,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "dataset_type": "s3", + "uri": "s3://compilation-job-bucket/output-config.tar.gz", + "target_device": "lambda", + "target_platform": "{'Os': 'ANDROID', 'Arch': 'X86_64', 'Accelerator': 'INTEL_GRAPHICS'}" + }, + "tags": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,labeling-job/data-source.tar_gz,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "dataset_type": "s3", - "uri": "s3://labeling-job/data-source.tar.gz" - }, - "externalUrl": null, - "description": null, - "uri": null, - "tags": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,edge-packaging-bucket/model-artifact.tar_gz,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "dataset_type": "s3", + "uri": "s3://edge-packaging-bucket/model-artifact.tar.gz" + }, + "tags": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,labeling-job/category-config.tar_gz,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "dataset_type": "s3", - "uri": "s3://labeling-job/category-config.tar.gz" - }, - "externalUrl": null, - "description": null, - "uri": null, - "tags": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,edge-packaging-bucket/output-config.tar_gz,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "dataset_type": "s3", + "uri": "s3://edge-packaging-bucket/output-config.tar.gz" + }, + "tags": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,labeling-job/output-dataset.tar_gz,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "dataset_type": "s3", - "uri": "s3://labeling-job/output-dataset.tar.gz" - }, - "externalUrl": null, - "description": null, - "uri": null, - "tags": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,labeling-job/data-source.tar_gz,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "dataset_type": "s3", + "uri": "s3://labeling-job/data-source.tar.gz" + }, + "tags": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,labeling-job/output-config.tar_gz,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "dataset_type": "s3", - "uri": "s3://labeling-job/output-config.tar.gz" - }, - "externalUrl": null, - "description": null, - "uri": null, - "tags": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,labeling-job/category-config.tar_gz,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "dataset_type": "s3", + "uri": "s3://labeling-job/category-config.tar.gz" + }, + "tags": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,processing-job/input-data.tar_gz,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "dataset_type": "s3", - "uri": "s3://processing-job/input-data.tar.gz", - "datatype": "ManifestFile", - "mode": "Pipe", - "distribution_type": "FullyReplicated", - "compression": "None", - "name": "string" - }, - "externalUrl": null, - "description": null, - "uri": null, - "tags": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,labeling-job/output-dataset.tar_gz,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "dataset_type": "s3", + "uri": "s3://labeling-job/output-dataset.tar.gz" + }, + "tags": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/input-dataset.tar_gz,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "dataset_type": "s3", - "uri": "s3://training-job/input-dataset.tar.gz", - "datatype": "None", - "distribution_type": "FullyReplicated", - "attribute_names": "['string']", - "channel_name": "string" - }, - "externalUrl": null, - "description": null, - "uri": null, - "tags": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,labeling-job/output-config.tar_gz,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "dataset_type": "s3", + "uri": "s3://labeling-job/output-config.tar.gz" + }, + "tags": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/output-data.tar_gz,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "dataset_type": "s3", - "uri": "s3://training-job/output-data.tar.gz" - }, - "externalUrl": null, - "description": null, - "uri": null, - "tags": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,processing-job/input-data.tar_gz,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "dataset_type": "s3", + "uri": "s3://processing-job/input-data.tar.gz", + "datatype": "ManifestFile", + "mode": "Pipe", + "distribution_type": "FullyReplicated", + "compression": "None", + "name": "string" + }, + "tags": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/checkpoint-config.tar_gz,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "dataset_type": "s3", - "uri": "s3://training-job/checkpoint-config.tar.gz" - }, - "externalUrl": null, - "description": null, - "uri": null, - "tags": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/input-dataset.tar_gz,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "dataset_type": "s3", + "uri": "s3://training-job/input-dataset.tar.gz", + "datatype": "None", + "distribution_type": "FullyReplicated", + "attribute_names": "['string']", + "channel_name": "string" + }, + "tags": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/debug-hook-config.tar_gz,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "dataset_type": "s3", - "uri": "s3://training-job/debug-hook-config.tar.gz" - }, - "externalUrl": null, - "description": null, - "uri": null, - "tags": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/output-data.tar_gz,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "dataset_type": "s3", + "uri": "s3://training-job/output-data.tar.gz" + }, + "tags": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/tensorboard-output-config.tar_gz,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "dataset_type": "s3", - "uri": "s3://training-job/tensorboard-output-config.tar.gz" - }, - "externalUrl": null, - "description": null, - "uri": null, - "tags": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/checkpoint-config.tar_gz,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "dataset_type": "s3", + "uri": "s3://training-job/checkpoint-config.tar.gz" + }, + "tags": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/profiler-config.tar_gz,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "dataset_type": "s3", - "uri": "s3://training-job/profiler-config.tar.gz" - }, - "externalUrl": null, - "description": null, - "uri": null, - "tags": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/debug-hook-config.tar_gz,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "dataset_type": "s3", + "uri": "s3://training-job/debug-hook-config.tar.gz" + }, + "tags": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/debug-rule-config.tar_gz,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "dataset_type": "s3", - "uri": "s3://training-job/debug-rule-config.tar.gz" - }, - "externalUrl": null, - "description": null, - "uri": null, - "tags": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/tensorboard-output-config.tar_gz,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "dataset_type": "s3", + "uri": "s3://training-job/tensorboard-output-config.tar.gz" + }, + "tags": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/profiler-rule-config.tar_gz,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "dataset_type": "s3", - "uri": "s3://training-job/profiler-rule-config.tar.gz" - }, - "externalUrl": null, - "description": null, - "uri": null, - "tags": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/profiler-config.tar_gz,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "dataset_type": "s3", + "uri": "s3://training-job/profiler-config.tar.gz" + }, + "tags": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,transform-job/input-data-source.tar_gz,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "dataset_type": "s3", - "uri": "s3://transform-job/input-data-source.tar.gz", - "datatype": "ManifestFile", - "compression": "None", - "split": "None" - }, - "externalUrl": null, - "description": null, - "uri": null, - "tags": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/debug-rule-config.tar_gz,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "dataset_type": "s3", + "uri": "s3://training-job/debug-rule-config.tar.gz" + }, + "tags": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,transform-job/output.tar_gz,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "dataset_type": "s3", - "uri": "s3://transform-job/output.tar.gz" - }, - "externalUrl": null, - "description": null, - "uri": null, - "tags": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/profiler-rule-config.tar_gz,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "dataset_type": "s3", + "uri": "s3://training-job/profiler-rule-config.tar.gz" + }, + "tags": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { - "urn": "urn:li:dataFlow:(sagemaker,auto_ml:an-auto-ml-job,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { - "customProperties": {}, - "externalUrl": null, - "name": "an-auto-ml-job", - "description": null, - "project": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,transform-job/input-data-source.tar_gz,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "dataset_type": "s3", + "uri": "s3://transform-job/input-data-source.tar.gz", + "datatype": "ManifestFile", + "compression": "None", + "split": "None" + }, + "tags": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,auto_ml:an-auto-ml-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:auto-ml-job/an-auto-ml-job)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "customProperties": { - "AutoMLJobName": "an-auto-ml-job", - "AutoMLJobArn": "arn:aws:sagemaker:us-west-2:123412341234:auto-ml-job/an-auto-ml-job", - "InputDataConfig": "[{'DataSource': {'S3DataSource': {'S3DataType': 'ManifestFile', 'S3Uri': 's3://auto-ml-job-input-bucket/file.txt'}}, 'CompressionType': 'None', 'TargetAttributeName': 'some-name'}]", - "OutputDataConfig": "{'KmsKeyId': 'some-key-id', 'S3OutputPath': 's3://auto-ml-job-output-bucket/file.txt'}", - "RoleArn": "arn:aws:iam::123412341234:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole", - "AutoMLJobObjective": "{'MetricName': 'Accuracy'}", - "ProblemType": "BinaryClassification", - "AutoMLJobConfig": "{'CompletionCriteria': {'MaxCandidates': 123, 'MaxRuntimePerTrainingJobInSeconds': 123, 'MaxAutoMLJobRuntimeInSeconds': 123}, 'SecurityConfig': {'VolumeKmsKeyId': 'string', 'EnableInterContainerTrafficEncryption': True, 'VpcConfig': {'SecurityGroupIds': ['string'], 'Subnets': ['string']}}}", - "CreationTime": "2015-01-01 00:00:00+00:00", - "EndTime": "2015-01-01 00:00:00+00:00", - "LastModifiedTime": "2015-01-01 00:00:00+00:00", - "FailureReason": "string", - "PartialFailureReasons": "[{'PartialFailureMessage': 'string'}]", - "BestCandidate": "{'CandidateName': 'string', 'FinalAutoMLJobObjectiveMetric': {'Type': 'Maximize', 'MetricName': 'Accuracy', 'Value': 1.0}, 'ObjectiveStatus': 'Succeeded', 'CandidateSteps': [{'CandidateStepType': 'AWS::SageMaker::TrainingJob', 'CandidateStepArn': 'string', 'CandidateStepName': 'string'}], 'CandidateStatus': 'Completed', 'InferenceContainers': [{'Image': 'string', 'ModelDataUrl': 's3://auto-ml-job/model-artifact.tar.gz', 'Environment': {'string': 'string'}}], 'CreationTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'EndTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'LastModifiedTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'FailureReason': 'string', 'CandidateProperties': {'CandidateArtifactLocations': {'Explainability': 'string'}}}", - "AutoMLJobStatus": "Completed", - "AutoMLJobSecondaryStatus": "Starting", - "GenerateCandidateDefinitionsOnly": "True", - "AutoMLJobArtifacts": "{'CandidateDefinitionNotebookLocation': 'string', 'DataExplorationNotebookLocation': 'string'}", - "ResolvedAttributes": "{'AutoMLJobObjective': {'MetricName': 'Accuracy'}, 'ProblemType': 'BinaryClassification', 'CompletionCriteria': {'MaxCandidates': 123, 'MaxRuntimePerTrainingJobInSeconds': 123, 'MaxAutoMLJobRuntimeInSeconds': 123}}", - "ModelDeployConfig": "{'AutoGenerateEndpointName': True, 'EndpointName': 'string'}", - "ModelDeployResult": "{'EndpointName': 'string'}", - "jobType": "auto_ml" - }, - "externalUrl": null, - "name": "an-auto-ml-job", - "description": null, - "type": { - "string": "SAGEMAKER" - }, - "flowUrn": null, - "status": "COMPLETED" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/auto_ml" - ] - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:s3,auto-ml-job-input-bucket/file_txt,PROD)" - ], - "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:s3,auto-ml-job-output-bucket/file_txt,PROD)" - ], - "inputDatajobs": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,transform-job/output.tar_gz,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "dataset_type": "s3", + "uri": "s3://transform-job/output.tar.gz" + }, + "tags": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { - "urn": "urn:li:dataFlow:(sagemaker,compilation:a-compilation-job,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { - "customProperties": {}, - "externalUrl": null, - "name": "a-compilation-job", - "description": null, - "project": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { + "urn": "urn:li:dataFlow:(sagemaker,auto_ml:an-auto-ml-job,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { + "customProperties": {}, + "name": "an-auto-ml-job" + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,compilation:a-compilation-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:compilation-job/a-compilation-job)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "customProperties": { - "CompilationJobName": "a-compilation-job", - "CompilationJobArn": "arn:aws:sagemaker:us-west-2:123412341234:compilation-job/a-compilation-job", - "CompilationJobStatus": "INPROGRESS", - "CompilationStartTime": "2015-01-01 00:00:00+00:00", - "CompilationEndTime": "2015-01-01 00:00:00+00:00", - "StoppingCondition": "{'MaxRuntimeInSeconds': 123, 'MaxWaitTimeInSeconds': 123}", - "InferenceImage": "string", - "CreationTime": "2015-01-01 00:00:00+00:00", - "LastModifiedTime": "2015-01-01 00:00:00+00:00", - "FailureReason": "string", - "ModelArtifacts": "{'S3ModelArtifacts': 's3://compilation-job-bucket/model-artifacts.tar.gz'}", - "ModelDigests": "{'ArtifactDigest': 'string'}", - "RoleArn": "arn:aws:iam::123412341234:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole", - "InputConfig": "{'S3Uri': 's3://compilation-job-bucket/input-config.tar.gz', 'DataInputConfig': 'string', 'Framework': 'TENSORFLOW', 'FrameworkVersion': 'string'}", - "OutputConfig": "{'S3OutputLocation': 's3://compilation-job-bucket/output-config.tar.gz', 'TargetDevice': 'lambda', 'TargetPlatform': {'Os': 'ANDROID', 'Arch': 'X86_64', 'Accelerator': 'INTEL_GRAPHICS'}, 'CompilerOptions': 'string', 'KmsKeyId': 'string'}", - "VpcConfig": "{'SecurityGroupIds': ['string'], 'Subnets': ['string']}", - "jobType": "compilation" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/compilation-jobs/a-compilation-job", - "name": "a-compilation-job", - "description": null, - "type": { - "string": "SAGEMAKER" - }, - "flowUrn": null, - "status": "IN_PROGRESS" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/compilation" - ] - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:s3,compilation-job-bucket/input-config.tar_gz,PROD)" - ], - "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:s3,compilation-job-bucket/output-config.tar_gz,PROD)" - ], - "inputDatajobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,compilation:a-compilation-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:compilation-job/a-compilation-job)" - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,auto_ml:an-auto-ml-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:auto-ml-job/an-auto-ml-job)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "AutoMLJobName": "an-auto-ml-job", + "AutoMLJobArn": "arn:aws:sagemaker:us-west-2:123412341234:auto-ml-job/an-auto-ml-job", + "InputDataConfig": "[{'DataSource': {'S3DataSource': {'S3DataType': 'ManifestFile', 'S3Uri': 's3://auto-ml-job-input-bucket/file.txt'}}, 'CompressionType': 'None', 'TargetAttributeName': 'some-name'}]", + "OutputDataConfig": "{'KmsKeyId': 'some-key-id', 'S3OutputPath': 's3://auto-ml-job-output-bucket/file.txt'}", + "RoleArn": "arn:aws:iam::123412341234:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole", + "AutoMLJobObjective": "{'MetricName': 'Accuracy'}", + "ProblemType": "BinaryClassification", + "AutoMLJobConfig": "{'CompletionCriteria': {'MaxCandidates': 123, 'MaxRuntimePerTrainingJobInSeconds': 123, 'MaxAutoMLJobRuntimeInSeconds': 123}, 'SecurityConfig': {'VolumeKmsKeyId': 'string', 'EnableInterContainerTrafficEncryption': True, 'VpcConfig': {'SecurityGroupIds': ['string'], 'Subnets': ['string']}}}", + "CreationTime": "2015-01-01 00:00:00+00:00", + "EndTime": "2015-01-01 00:00:00+00:00", + "LastModifiedTime": "2015-01-01 00:00:00+00:00", + "FailureReason": "string", + "PartialFailureReasons": "[{'PartialFailureMessage': 'string'}]", + "BestCandidate": "{'CandidateName': 'string', 'FinalAutoMLJobObjectiveMetric': {'Type': 'Maximize', 'MetricName': 'Accuracy', 'Value': 1.0}, 'ObjectiveStatus': 'Succeeded', 'CandidateSteps': [{'CandidateStepType': 'AWS::SageMaker::TrainingJob', 'CandidateStepArn': 'string', 'CandidateStepName': 'string'}], 'CandidateStatus': 'Completed', 'InferenceContainers': [{'Image': 'string', 'ModelDataUrl': 's3://auto-ml-job/model-artifact.tar.gz', 'Environment': {'string': 'string'}}], 'CreationTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'EndTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'LastModifiedTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'FailureReason': 'string', 'CandidateProperties': {'CandidateArtifactLocations': {'Explainability': 'string'}}}", + "AutoMLJobStatus": "Completed", + "AutoMLJobSecondaryStatus": "Starting", + "GenerateCandidateDefinitionsOnly": "True", + "AutoMLJobArtifacts": "{'CandidateDefinitionNotebookLocation': 'string', 'DataExplorationNotebookLocation': 'string'}", + "ResolvedAttributes": "{'AutoMLJobObjective': {'MetricName': 'Accuracy'}, 'ProblemType': 'BinaryClassification', 'CompletionCriteria': {'MaxCandidates': 123, 'MaxRuntimePerTrainingJobInSeconds': 123, 'MaxAutoMLJobRuntimeInSeconds': 123}}", + "ModelDeployConfig": "{'AutoGenerateEndpointName': True, 'EndpointName': 'string'}", + "ModelDeployResult": "{'EndpointName': 'string'}", + "jobType": "auto_ml" + }, + "name": "an-auto-ml-job", + "type": { + "string": "SAGEMAKER" + }, + "status": "COMPLETED" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/auto_ml" + ] + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:s3,auto-ml-job-input-bucket/file_txt,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:s3,auto-ml-job-output-bucket/file_txt,PROD)" + ], + "inputDatajobs": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { - "urn": "urn:li:dataFlow:(sagemaker,edge_packaging:an-edge-packaging-job,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { - "customProperties": {}, - "externalUrl": null, - "name": "an-edge-packaging-job", - "description": null, - "project": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { + "urn": "urn:li:dataFlow:(sagemaker,compilation:a-compilation-job,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { + "customProperties": {}, + "name": "a-compilation-job" + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,edge_packaging:an-edge-packaging-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:edge-packaging-job/an-edge-packaging-job)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "customProperties": { - "EdgePackagingJobArn": "arn:aws:sagemaker:us-west-2:123412341234:edge-packaging-job/an-edge-packaging-job", - "EdgePackagingJobName": "an-edge-packaging-job", - "CompilationJobName": "a-compilation-job", - "ModelName": "the-second-model", - "ModelVersion": "string", - "RoleArn": "arn:aws:iam::123412341234:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole", - "OutputConfig": "{'S3OutputLocation': 's3://edge-packaging-bucket/output-config.tar.gz', 'KmsKeyId': 'string', 'PresetDeploymentType': 'GreengrassV2Component', 'PresetDeploymentConfig': 'string'}", - "ResourceKey": "string", - "EdgePackagingJobStatus": "STARTING", - "EdgePackagingJobStatusMessage": "string", - "CreationTime": "2015-01-01 00:00:00+00:00", - "LastModifiedTime": "2015-01-01 00:00:00+00:00", - "ModelArtifact": "s3://edge-packaging-bucket/model-artifact.tar.gz", - "ModelSignature": "string", - "PresetDeploymentOutput": "{'Type': 'GreengrassV2Component', 'Artifact': 'arn:aws:sagemaker:us-west-2:123412341234:edge-packaging-job/some-artifact', 'Status': 'COMPLETED', 'StatusMessage': 'string'}", - "jobType": "edge_packaging" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/edge-packaging-jobs/an-edge-packaging-job", - "name": "an-edge-packaging-job", - "description": null, - "type": { - "string": "SAGEMAKER" - }, - "flowUrn": null, - "status": "STARTING" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/edge_packaging" - ] - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [], - "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:s3,edge-packaging-bucket/model-artifact.tar_gz,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:s3,edge-packaging-bucket/output-config.tar_gz,PROD)" - ], - "inputDatajobs": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,compilation:a-compilation-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:compilation-job/a-compilation-job)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "CompilationJobName": "a-compilation-job", + "CompilationJobArn": "arn:aws:sagemaker:us-west-2:123412341234:compilation-job/a-compilation-job", + "CompilationJobStatus": "INPROGRESS", + "CompilationStartTime": "2015-01-01 00:00:00+00:00", + "CompilationEndTime": "2015-01-01 00:00:00+00:00", + "StoppingCondition": "{'MaxRuntimeInSeconds': 123, 'MaxWaitTimeInSeconds': 123}", + "InferenceImage": "string", + "CreationTime": "2015-01-01 00:00:00+00:00", + "LastModifiedTime": "2015-01-01 00:00:00+00:00", + "FailureReason": "string", + "ModelArtifacts": "{'S3ModelArtifacts': 's3://compilation-job-bucket/model-artifacts.tar.gz'}", + "ModelDigests": "{'ArtifactDigest': 'string'}", + "RoleArn": "arn:aws:iam::123412341234:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole", + "InputConfig": "{'S3Uri': 's3://compilation-job-bucket/input-config.tar.gz', 'DataInputConfig': 'string', 'Framework': 'TENSORFLOW', 'FrameworkVersion': 'string'}", + "OutputConfig": "{'S3OutputLocation': 's3://compilation-job-bucket/output-config.tar.gz', 'TargetDevice': 'lambda', 'TargetPlatform': {'Os': 'ANDROID', 'Arch': 'X86_64', 'Accelerator': 'INTEL_GRAPHICS'}, 'CompilerOptions': 'string', 'KmsKeyId': 'string'}", + "VpcConfig": "{'SecurityGroupIds': ['string'], 'Subnets': ['string']}", + "jobType": "compilation" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/compilation-jobs/a-compilation-job", + "name": "a-compilation-job", + "type": { + "string": "SAGEMAKER" + }, + "status": "IN_PROGRESS" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/compilation" + ] + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:s3,compilation-job-bucket/input-config.tar_gz,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:s3,compilation-job-bucket/output-config.tar_gz,PROD)" + ], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,compilation:a-compilation-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:compilation-job/a-compilation-job)" + ] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { - "urn": "urn:li:dataFlow:(sagemaker,hyper_parameter_tuning:a-hyper-parameter-tuning-job,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { - "customProperties": {}, - "externalUrl": null, - "name": "a-hyper-parameter-tuning-job", - "description": null, - "project": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { + "urn": "urn:li:dataFlow:(sagemaker,edge_packaging:an-edge-packaging-job,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { + "customProperties": {}, + "name": "an-edge-packaging-job" + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,hyper_parameter_tuning:a-hyper-parameter-tuning-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:hyper-parameter-tuning-job/a-hyper-parameter-tuning-job)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "customProperties": { - "HyperParameterTuningJobName": "a-hyper-parameter-tuning-job", - "HyperParameterTuningJobArn": "arn:aws:sagemaker:us-west-2:123412341234:hyper-parameter-tuning-job/a-hyper-parameter-tuning-job", - "HyperParameterTuningJobConfig": "{'Strategy': 'Bayesian', 'HyperParameterTuningJobObjective': {'Type': 'Maximize', 'MetricName': 'string'}, 'ResourceLimits': {'MaxNumberOfTrainingJobs': 123, 'MaxParallelTrainingJobs': 123}, 'ParameterRanges': {'IntegerParameterRanges': [{'Name': 'string', 'MinValue': 'string', 'MaxValue': 'string', 'ScalingType': 'Auto'}], 'ContinuousParameterRanges': [{'Name': 'string', 'MinValue': 'string', 'MaxValue': 'string', 'ScalingType': 'Auto'}], 'CategoricalParameterRanges': [{'Name': 'string', 'Values': ['string']}]}, 'TrainingJobEarlyStoppingType': 'Off', 'TuningJobCompletionCriteria': {'TargetObjectiveMetricValue': 1.0}}", - "TrainingJobDefinition": "{'DefinitionName': 'string', 'TuningObjective': {'Type': 'Maximize', 'MetricName': 'string'}, 'HyperParameterRanges': {'IntegerParameterRanges': [{'Name': 'string', 'MinValue': 'string', 'MaxValue': 'string', 'ScalingType': 'Auto'}], 'ContinuousParameterRanges': [{'Name': 'string', 'MinValue': 'string', 'MaxValue': 'string', 'ScalingType': 'Auto'}], 'CategoricalParameterRanges': [{'Name': 'string', 'Values': ['string']}]}, 'StaticHyperParameters': {'string': 'string'}, 'AlgorithmSpecification': {'TrainingImage': 'string', 'TrainingInputMode': 'Pipe', 'AlgorithmName': 'string', 'MetricDefinitions': [{'Name': 'string', 'Regex': 'string'}]}, 'RoleArn': 'arn:aws:iam::123412341234:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole', 'InputDataConfig': [{'ChannelName': 'string', 'DataSource': {'S3DataSource': {'S3DataType': 'ManifestFile', 'S3Uri': 's3://hyper-parameter-tuning-job/data-source.tar.gz', 'S3DataDistributionType': 'FullyReplicated', 'AttributeNames': ['string']}, 'FileSystemDataSource': {'FileSystemId': 'abcdefgihjklmnopqrstuvwxyz', 'FileSystemAccessMode': 'rw', 'FileSystemType': 'EFS', 'DirectoryPath': 'string'}}, 'ContentType': 'string', 'CompressionType': 'None', 'RecordWrapperType': 'None', 'InputMode': 'Pipe', 'ShuffleConfig': {'Seed': 123}}], 'VpcConfig': {'SecurityGroupIds': ['string'], 'Subnets': ['string']}, 'OutputDataConfig': {'KmsKeyId': 'string', 'S3OutputPath': 's3://hyper-parameter-tuning-job/data-output.tar.gz'}, 'ResourceConfig': {'InstanceType': 'ml.m4.xlarge', 'InstanceCount': 123, 'VolumeSizeInGB': 123, 'VolumeKmsKeyId': 'string'}, 'StoppingCondition': {'MaxRuntimeInSeconds': 123, 'MaxWaitTimeInSeconds': 123}, 'EnableNetworkIsolation': True, 'EnableInterContainerTrafficEncryption': True, 'EnableManagedSpotTraining': True, 'CheckpointConfig': {'S3Uri': 's3://hyper-parameter-tuning-job/checkpoint-config.tar.gz', 'LocalPath': 'string'}, 'RetryStrategy': {'MaximumRetryAttempts': 123}}", - "TrainingJobDefinitions": "[{'DefinitionName': 'string', 'TuningObjective': {'Type': 'Maximize', 'MetricName': 'string'}, 'HyperParameterRanges': {'IntegerParameterRanges': [{'Name': 'string', 'MinValue': 'string', 'MaxValue': 'string', 'ScalingType': 'Auto'}], 'ContinuousParameterRanges': [{'Name': 'string', 'MinValue': 'string', 'MaxValue': 'string', 'ScalingType': 'Auto'}], 'CategoricalParameterRanges': [{'Name': 'string', 'Values': ['string']}]}, 'StaticHyperParameters': {'string': 'string'}, 'AlgorithmSpecification': {'TrainingImage': 'string', 'TrainingInputMode': 'Pipe', 'AlgorithmName': 'string', 'MetricDefinitions': [{'Name': 'string', 'Regex': 'string'}]}, 'RoleArn': 'arn:aws:iam::123412341234:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole', 'InputDataConfig': [{'ChannelName': 'string', 'DataSource': {'S3DataSource': {'S3DataType': 'ManifestFile', 'S3Uri': 's3://hyper-parameter-tuning-job/data-source.tar.gz', 'S3DataDistributionType': 'FullyReplicated', 'AttributeNames': ['string']}, 'FileSystemDataSource': {'FileSystemId': 'abcdefgihjklmnopqrstuvwxyz', 'FileSystemAccessMode': 'rw', 'FileSystemType': 'EFS', 'DirectoryPath': 'string'}}, 'ContentType': 'string', 'CompressionType': 'None', 'RecordWrapperType': 'None', 'InputMode': 'Pipe', 'ShuffleConfig': {'Seed': 123}}], 'VpcConfig': {'SecurityGroupIds': ['string'], 'Subnets': ['string']}, 'OutputDataConfig': {'KmsKeyId': 'string', 'S3OutputPath': 's3://hyper-parameter-tuning-job/data-output.tar.gz'}, 'ResourceConfig': {'InstanceType': 'ml.m4.xlarge', 'InstanceCount': 123, 'VolumeSizeInGB': 123, 'VolumeKmsKeyId': 'string'}, 'StoppingCondition': {'MaxRuntimeInSeconds': 123, 'MaxWaitTimeInSeconds': 123}, 'EnableNetworkIsolation': True, 'EnableInterContainerTrafficEncryption': True, 'EnableManagedSpotTraining': True, 'CheckpointConfig': {'S3Uri': 's3://hyper-parameter-tuning-job/checkpoint-config.tar.gz', 'LocalPath': 'string'}, 'RetryStrategy': {'MaximumRetryAttempts': 123}}]", - "HyperParameterTuningJobStatus": "Completed", - "CreationTime": "2015-01-01 00:00:00+00:00", - "HyperParameterTuningEndTime": "2015-01-01 00:00:00+00:00", - "LastModifiedTime": "2015-01-01 00:00:00+00:00", - "TrainingJobStatusCounters": "{'Completed': 123, 'InProgress': 123, 'RetryableError': 123, 'NonRetryableError': 123, 'Stopped': 123}", - "ObjectiveStatusCounters": "{'Succeeded': 123, 'Pending': 123, 'Failed': 123}", - "BestTrainingJob": "{'TrainingJobDefinitionName': 'string', 'TrainingJobName': 'string', 'TrainingJobArn': 'string', 'TuningJobName': 'string', 'CreationTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'TrainingStartTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'TrainingEndTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'TrainingJobStatus': 'InProgress', 'TunedHyperParameters': {'string': 'string'}, 'FailureReason': 'string', 'FinalHyperParameterTuningJobObjectiveMetric': {'Type': 'Maximize', 'MetricName': 'string', 'Value': 1.0}, 'ObjectiveStatus': 'Succeeded'}", - "OverallBestTrainingJob": "{'TrainingJobDefinitionName': 'string', 'TrainingJobName': 'string', 'TrainingJobArn': 'string', 'TuningJobName': 'string', 'CreationTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'TrainingStartTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'TrainingEndTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'TrainingJobStatus': 'InProgress', 'TunedHyperParameters': {'string': 'string'}, 'FailureReason': 'string', 'FinalHyperParameterTuningJobObjectiveMetric': {'Type': 'Maximize', 'MetricName': 'string', 'Value': 1.0}, 'ObjectiveStatus': 'Succeeded'}", - "WarmStartConfig": "{'ParentHyperParameterTuningJobs': [{'HyperParameterTuningJobName': 'string'}], 'WarmStartType': 'IdenticalDataAndAlgorithm'}", - "FailureReason": "string", - "jobType": "hyper_parameter_tuning" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/hyper-tuning-jobs/a-hyper-parameter-tuning-job", - "name": "a-hyper-parameter-tuning-job", - "description": null, - "type": { - "string": "SAGEMAKER" - }, - "flowUrn": null, - "status": "COMPLETED" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/hyper_parameter_tuning" - ] - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [], - "outputDatasets": [], - "inputDatajobs": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,edge_packaging:an-edge-packaging-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:edge-packaging-job/an-edge-packaging-job)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "EdgePackagingJobArn": "arn:aws:sagemaker:us-west-2:123412341234:edge-packaging-job/an-edge-packaging-job", + "EdgePackagingJobName": "an-edge-packaging-job", + "CompilationJobName": "a-compilation-job", + "ModelName": "the-second-model", + "ModelVersion": "string", + "RoleArn": "arn:aws:iam::123412341234:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole", + "OutputConfig": "{'S3OutputLocation': 's3://edge-packaging-bucket/output-config.tar.gz', 'KmsKeyId': 'string', 'PresetDeploymentType': 'GreengrassV2Component', 'PresetDeploymentConfig': 'string'}", + "ResourceKey": "string", + "EdgePackagingJobStatus": "STARTING", + "EdgePackagingJobStatusMessage": "string", + "CreationTime": "2015-01-01 00:00:00+00:00", + "LastModifiedTime": "2015-01-01 00:00:00+00:00", + "ModelArtifact": "s3://edge-packaging-bucket/model-artifact.tar.gz", + "ModelSignature": "string", + "PresetDeploymentOutput": "{'Type': 'GreengrassV2Component', 'Artifact': 'arn:aws:sagemaker:us-west-2:123412341234:edge-packaging-job/some-artifact', 'Status': 'COMPLETED', 'StatusMessage': 'string'}", + "jobType": "edge_packaging" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/edge-packaging-jobs/an-edge-packaging-job", + "name": "an-edge-packaging-job", + "type": { + "string": "SAGEMAKER" + }, + "status": "STARTING" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/edge_packaging" + ] + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:s3,edge-packaging-bucket/model-artifact.tar_gz,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:s3,edge-packaging-bucket/output-config.tar_gz,PROD)" + ], + "inputDatajobs": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { - "urn": "urn:li:dataFlow:(sagemaker,labeling:a-labeling-job,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { - "customProperties": {}, - "externalUrl": null, - "name": "a-labeling-job", - "description": null, - "project": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { + "urn": "urn:li:dataFlow:(sagemaker,hyper_parameter_tuning:a-hyper-parameter-tuning-job,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { + "customProperties": {}, + "name": "a-hyper-parameter-tuning-job" + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,labeling:a-labeling-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:labeling-job/a-labeling-job)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "customProperties": { - "LabelingJobStatus": "Initializing", - "LabelCounters": "{'TotalLabeled': 123, 'HumanLabeled': 123, 'MachineLabeled': 123, 'FailedNonRetryableError': 123, 'Unlabeled': 123}", - "FailureReason": "string", - "CreationTime": "2015-01-01 00:00:00+00:00", - "LastModifiedTime": "2015-01-01 00:00:00+00:00", - "JobReferenceCode": "string", - "LabelingJobName": "a-labeling-job", - "LabelingJobArn": "arn:aws:sagemaker:us-west-2:123412341234:labeling-job/a-labeling-job", - "LabelAttributeName": "string", - "InputConfig": "{'DataSource': {'S3DataSource': {'ManifestS3Uri': 's3://labeling-job/data-source.tar.gz'}, 'SnsDataSource': {'SnsTopicArn': 'string'}}, 'DataAttributes': {'ContentClassifiers': ['FreeOfPersonallyIdentifiableInformation', 'FreeOfAdultContent']}}", - "OutputConfig": "{'S3OutputPath': 's3://labeling-job/output-config.tar.gz', 'KmsKeyId': 'string', 'SnsTopicArn': 'string'}", - "RoleArn": "arn:aws:iam::123412341234:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole", - "LabelCategoryConfigS3Uri": "s3://labeling-job/category-config.tar.gz", - "StoppingConditions": "{'MaxHumanLabeledObjectCount': 123, 'MaxPercentageOfInputDatasetLabeled': 123}", - "LabelingJobAlgorithmsConfig": "{'LabelingJobAlgorithmSpecificationArn': 'string', 'InitialActiveLearningModelArn': 'arn:aws:sagemaker:us-west-2:123412341234:labeling-job/initial-active-learning-model', 'LabelingJobResourceConfig': {'VolumeKmsKeyId': 'string'}}", - "HumanTaskConfig": "{'WorkteamArn': 'string', 'UiConfig': {'UiTemplateS3Uri': 's3://labeling-job/ui-config.tar.gz', 'HumanTaskUiArn': 'string'}, 'PreHumanTaskLambdaArn': 'string', 'TaskKeywords': ['string'], 'TaskTitle': 'string', 'TaskDescription': 'string', 'NumberOfHumanWorkersPerDataObject': 123, 'TaskTimeLimitInSeconds': 123, 'TaskAvailabilityLifetimeInSeconds': 123, 'MaxConcurrentTaskCount': 123, 'AnnotationConsolidationConfig': {'AnnotationConsolidationLambdaArn': 'string'}, 'PublicWorkforceTaskPrice': {'AmountInUsd': {'Dollars': 123, 'Cents': 123, 'TenthFractionsOfACent': 123}}}", - "Tags": "[{'Key': 'string', 'Value': 'string'}]", - "LabelingJobOutput": "{'OutputDatasetS3Uri': 's3://labeling-job/output-dataset.tar.gz', 'FinalActiveLearningModelArn': 'arn:aws:sagemaker:us-west-2:123412341234:labeling-job/final-active-learning-model'}", - "jobType": "labeling" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/labeling-jobs/a-labeling-job", - "name": "a-labeling-job", - "description": null, - "type": { - "string": "SAGEMAKER" - }, - "flowUrn": null, - "status": "STARTING" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/labeling" - ] - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:s3,labeling-job/category-config.tar_gz,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:s3,labeling-job/data-source.tar_gz,PROD)" - ], - "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:s3,labeling-job/output-config.tar_gz,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:s3,labeling-job/output-dataset.tar_gz,PROD)" - ], - "inputDatajobs": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,hyper_parameter_tuning:a-hyper-parameter-tuning-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:hyper-parameter-tuning-job/a-hyper-parameter-tuning-job)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "HyperParameterTuningJobName": "a-hyper-parameter-tuning-job", + "HyperParameterTuningJobArn": "arn:aws:sagemaker:us-west-2:123412341234:hyper-parameter-tuning-job/a-hyper-parameter-tuning-job", + "HyperParameterTuningJobConfig": "{'Strategy': 'Bayesian', 'HyperParameterTuningJobObjective': {'Type': 'Maximize', 'MetricName': 'string'}, 'ResourceLimits': {'MaxNumberOfTrainingJobs': 123, 'MaxParallelTrainingJobs': 123}, 'ParameterRanges': {'IntegerParameterRanges': [{'Name': 'string', 'MinValue': 'string', 'MaxValue': 'string', 'ScalingType': 'Auto'}], 'ContinuousParameterRanges': [{'Name': 'string', 'MinValue': 'string', 'MaxValue': 'string', 'ScalingType': 'Auto'}], 'CategoricalParameterRanges': [{'Name': 'string', 'Values': ['string']}]}, 'TrainingJobEarlyStoppingType': 'Off', 'TuningJobCompletionCriteria': {'TargetObjectiveMetricValue': 1.0}}", + "TrainingJobDefinition": "{'DefinitionName': 'string', 'TuningObjective': {'Type': 'Maximize', 'MetricName': 'string'}, 'HyperParameterRanges': {'IntegerParameterRanges': [{'Name': 'string', 'MinValue': 'string', 'MaxValue': 'string', 'ScalingType': 'Auto'}], 'ContinuousParameterRanges': [{'Name': 'string', 'MinValue': 'string', 'MaxValue': 'string', 'ScalingType': 'Auto'}], 'CategoricalParameterRanges': [{'Name': 'string', 'Values': ['string']}]}, 'StaticHyperParameters': {'string': 'string'}, 'AlgorithmSpecification': {'TrainingImage': 'string', 'TrainingInputMode': 'Pipe', 'AlgorithmName': 'string', 'MetricDefinitions': [{'Name': 'string', 'Regex': 'string'}]}, 'RoleArn': 'arn:aws:iam::123412341234:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole', 'InputDataConfig': [{'ChannelName': 'string', 'DataSource': {'S3DataSource': {'S3DataType': 'ManifestFile', 'S3Uri': 's3://hyper-parameter-tuning-job/data-source.tar.gz', 'S3DataDistributionType': 'FullyReplicated', 'AttributeNames': ['string']}, 'FileSystemDataSource': {'FileSystemId': 'abcdefgihjklmnopqrstuvwxyz', 'FileSystemAccessMode': 'rw', 'FileSystemType': 'EFS', 'DirectoryPath': 'string'}}, 'ContentType': 'string', 'CompressionType': 'None', 'RecordWrapperType': 'None', 'InputMode': 'Pipe', 'ShuffleConfig': {'Seed': 123}}], 'VpcConfig': {'SecurityGroupIds': ['string'], 'Subnets': ['string']}, 'OutputDataConfig': {'KmsKeyId': 'string', 'S3OutputPath': 's3://hyper-parameter-tuning-job/data-output.tar.gz'}, 'ResourceConfig': {'InstanceType': 'ml.m4.xlarge', 'InstanceCount': 123, 'VolumeSizeInGB': 123, 'VolumeKmsKeyId': 'string'}, 'StoppingCondition': {'MaxRuntimeInSeconds': 123, 'MaxWaitTimeInSeconds': 123}, 'EnableNetworkIsolation': True, 'EnableInterContainerTrafficEncryption': True, 'EnableManagedSpotTraining': True, 'CheckpointConfig': {'S3Uri': 's3://hyper-parameter-tuning-job/checkpoint-config.tar.gz', 'LocalPath': 'string'}, 'RetryStrategy': {'MaximumRetryAttempts': 123}}", + "TrainingJobDefinitions": "[{'DefinitionName': 'string', 'TuningObjective': {'Type': 'Maximize', 'MetricName': 'string'}, 'HyperParameterRanges': {'IntegerParameterRanges': [{'Name': 'string', 'MinValue': 'string', 'MaxValue': 'string', 'ScalingType': 'Auto'}], 'ContinuousParameterRanges': [{'Name': 'string', 'MinValue': 'string', 'MaxValue': 'string', 'ScalingType': 'Auto'}], 'CategoricalParameterRanges': [{'Name': 'string', 'Values': ['string']}]}, 'StaticHyperParameters': {'string': 'string'}, 'AlgorithmSpecification': {'TrainingImage': 'string', 'TrainingInputMode': 'Pipe', 'AlgorithmName': 'string', 'MetricDefinitions': [{'Name': 'string', 'Regex': 'string'}]}, 'RoleArn': 'arn:aws:iam::123412341234:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole', 'InputDataConfig': [{'ChannelName': 'string', 'DataSource': {'S3DataSource': {'S3DataType': 'ManifestFile', 'S3Uri': 's3://hyper-parameter-tuning-job/data-source.tar.gz', 'S3DataDistributionType': 'FullyReplicated', 'AttributeNames': ['string']}, 'FileSystemDataSource': {'FileSystemId': 'abcdefgihjklmnopqrstuvwxyz', 'FileSystemAccessMode': 'rw', 'FileSystemType': 'EFS', 'DirectoryPath': 'string'}}, 'ContentType': 'string', 'CompressionType': 'None', 'RecordWrapperType': 'None', 'InputMode': 'Pipe', 'ShuffleConfig': {'Seed': 123}}], 'VpcConfig': {'SecurityGroupIds': ['string'], 'Subnets': ['string']}, 'OutputDataConfig': {'KmsKeyId': 'string', 'S3OutputPath': 's3://hyper-parameter-tuning-job/data-output.tar.gz'}, 'ResourceConfig': {'InstanceType': 'ml.m4.xlarge', 'InstanceCount': 123, 'VolumeSizeInGB': 123, 'VolumeKmsKeyId': 'string'}, 'StoppingCondition': {'MaxRuntimeInSeconds': 123, 'MaxWaitTimeInSeconds': 123}, 'EnableNetworkIsolation': True, 'EnableInterContainerTrafficEncryption': True, 'EnableManagedSpotTraining': True, 'CheckpointConfig': {'S3Uri': 's3://hyper-parameter-tuning-job/checkpoint-config.tar.gz', 'LocalPath': 'string'}, 'RetryStrategy': {'MaximumRetryAttempts': 123}}]", + "HyperParameterTuningJobStatus": "Completed", + "CreationTime": "2015-01-01 00:00:00+00:00", + "HyperParameterTuningEndTime": "2015-01-01 00:00:00+00:00", + "LastModifiedTime": "2015-01-01 00:00:00+00:00", + "TrainingJobStatusCounters": "{'Completed': 123, 'InProgress': 123, 'RetryableError': 123, 'NonRetryableError': 123, 'Stopped': 123}", + "ObjectiveStatusCounters": "{'Succeeded': 123, 'Pending': 123, 'Failed': 123}", + "BestTrainingJob": "{'TrainingJobDefinitionName': 'string', 'TrainingJobName': 'string', 'TrainingJobArn': 'string', 'TuningJobName': 'string', 'CreationTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'TrainingStartTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'TrainingEndTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'TrainingJobStatus': 'InProgress', 'TunedHyperParameters': {'string': 'string'}, 'FailureReason': 'string', 'FinalHyperParameterTuningJobObjectiveMetric': {'Type': 'Maximize', 'MetricName': 'string', 'Value': 1.0}, 'ObjectiveStatus': 'Succeeded'}", + "OverallBestTrainingJob": "{'TrainingJobDefinitionName': 'string', 'TrainingJobName': 'string', 'TrainingJobArn': 'string', 'TuningJobName': 'string', 'CreationTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'TrainingStartTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'TrainingEndTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'TrainingJobStatus': 'InProgress', 'TunedHyperParameters': {'string': 'string'}, 'FailureReason': 'string', 'FinalHyperParameterTuningJobObjectiveMetric': {'Type': 'Maximize', 'MetricName': 'string', 'Value': 1.0}, 'ObjectiveStatus': 'Succeeded'}", + "WarmStartConfig": "{'ParentHyperParameterTuningJobs': [{'HyperParameterTuningJobName': 'string'}], 'WarmStartType': 'IdenticalDataAndAlgorithm'}", + "FailureReason": "string", + "jobType": "hyper_parameter_tuning" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/hyper-tuning-jobs/a-hyper-parameter-tuning-job", + "name": "a-hyper-parameter-tuning-job", + "type": { + "string": "SAGEMAKER" + }, + "status": "COMPLETED" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/hyper_parameter_tuning" + ] + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { - "urn": "urn:li:dataFlow:(sagemaker,processing:a-processing-job,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { - "customProperties": {}, - "externalUrl": null, - "name": "a-processing-job", - "description": null, - "project": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { + "urn": "urn:li:dataFlow:(sagemaker,labeling:a-labeling-job,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { + "customProperties": {}, + "name": "a-labeling-job" + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,processing:a-processing-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:processing-job/a-processing-job)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "customProperties": { - "ProcessingJobName": "a-processing-job", - "ProcessingJobArn": "arn:aws:sagemaker:us-west-2:123412341234:processing-job/a-processing-job", - "ProcessingInputs": "[{'InputName': 'string', 'AppManaged': True, 'S3Input': {'S3Uri': 's3://processing-job/input-data.tar.gz', 'LocalPath': 'string', 'S3DataType': 'ManifestFile', 'S3InputMode': 'Pipe', 'S3DataDistributionType': 'FullyReplicated', 'S3CompressionType': 'None'}, 'DatasetDefinition': {'AthenaDatasetDefinition': {'Catalog': 'athena-catalog', 'Database': 'athena-database', 'QueryString': 'athena-query-string', 'WorkGroup': 'athena-work-group', 'OutputS3Uri': 's3://processing-job/athena-output.tar.gz', 'KmsKeyId': 'string', 'OutputFormat': 'PARQUET', 'OutputCompression': 'GZIP'}, 'RedshiftDatasetDefinition': {'ClusterId': 'redshift-cluster', 'Database': 'redshift-database', 'DbUser': 'redshift-db-user', 'QueryString': 'redshift-query-string', 'ClusterRoleArn': 'arn:aws:sagemaker:us-west-2:123412341234:processing-job/redshift-cluster', 'OutputS3Uri': 's3://processing-job/redshift-output.tar.gz', 'KmsKeyId': 'string', 'OutputFormat': 'PARQUET', 'OutputCompression': 'None'}, 'LocalPath': 'string', 'DataDistributionType': 'FullyReplicated', 'InputMode': 'Pipe'}}]", - "ProcessingOutputConfig": "{'Outputs': [{'OutputName': 'string', 'S3Output': {'S3Uri': 's3://processing-job/processing-output.tar.gz', 'LocalPath': 'string', 'S3UploadMode': 'Continuous'}, 'FeatureStoreOutput': {'FeatureGroupName': 'string'}, 'AppManaged': True}], 'KmsKeyId': 'string'}", - "ProcessingResources": "{'ClusterConfig': {'InstanceCount': 123, 'InstanceType': 'ml.t3.medium', 'VolumeSizeInGB': 123, 'VolumeKmsKeyId': 'string'}}", - "StoppingCondition": "{'MaxRuntimeInSeconds': 123}", - "AppSpecification": "{'ImageUri': 'string', 'ContainerEntrypoint': ['string'], 'ContainerArguments': ['string']}", - "Environment": "{'string': 'string'}", - "NetworkConfig": "{'EnableInterContainerTrafficEncryption': True, 'EnableNetworkIsolation': True, 'VpcConfig': {'SecurityGroupIds': ['string'], 'Subnets': ['string']}}", - "RoleArn": "arn:aws:iam::123412341234:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole", - "ExperimentConfig": "{'ExperimentName': 'string', 'TrialName': 'string', 'TrialComponentDisplayName': 'string'}", - "ProcessingJobStatus": "InProgress", - "ExitMessage": "string", - "FailureReason": "string", - "ProcessingEndTime": "2015-01-01 00:00:00+00:00", - "ProcessingStartTime": "2015-01-01 00:00:00+00:00", - "LastModifiedTime": "2015-01-01 00:00:00+00:00", - "CreationTime": "2015-01-01 00:00:00+00:00", - "MonitoringScheduleArn": "string", - "AutoMLJobArn": "arn:aws:sagemaker:us-west-2:123412341234:auto-ml-job/an-auto-ml-job", - "TrainingJobArn": "arn:aws:sagemaker:us-west-2:123412341234:training-job/a-training-job", - "jobType": "processing" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/processing-jobs/a-processing-job", - "name": "a-processing-job", - "description": null, - "type": { - "string": "SAGEMAKER" - }, - "flowUrn": null, - "status": "IN_PROGRESS" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/processing" - ] - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:s3,processing-job/input-data.tar_gz,PROD)" - ], - "outputDatasets": [], - "inputDatajobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,auto_ml:an-auto-ml-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:auto-ml-job/an-auto-ml-job)", - "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,training:a-training-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:training-job/a-training-job)" - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,labeling:a-labeling-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:labeling-job/a-labeling-job)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "LabelingJobStatus": "Initializing", + "LabelCounters": "{'TotalLabeled': 123, 'HumanLabeled': 123, 'MachineLabeled': 123, 'FailedNonRetryableError': 123, 'Unlabeled': 123}", + "FailureReason": "string", + "CreationTime": "2015-01-01 00:00:00+00:00", + "LastModifiedTime": "2015-01-01 00:00:00+00:00", + "JobReferenceCode": "string", + "LabelingJobName": "a-labeling-job", + "LabelingJobArn": "arn:aws:sagemaker:us-west-2:123412341234:labeling-job/a-labeling-job", + "LabelAttributeName": "string", + "InputConfig": "{'DataSource': {'S3DataSource': {'ManifestS3Uri': 's3://labeling-job/data-source.tar.gz'}, 'SnsDataSource': {'SnsTopicArn': 'string'}}, 'DataAttributes': {'ContentClassifiers': ['FreeOfPersonallyIdentifiableInformation', 'FreeOfAdultContent']}}", + "OutputConfig": "{'S3OutputPath': 's3://labeling-job/output-config.tar.gz', 'KmsKeyId': 'string', 'SnsTopicArn': 'string'}", + "RoleArn": "arn:aws:iam::123412341234:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole", + "LabelCategoryConfigS3Uri": "s3://labeling-job/category-config.tar.gz", + "StoppingConditions": "{'MaxHumanLabeledObjectCount': 123, 'MaxPercentageOfInputDatasetLabeled': 123}", + "LabelingJobAlgorithmsConfig": "{'LabelingJobAlgorithmSpecificationArn': 'string', 'InitialActiveLearningModelArn': 'arn:aws:sagemaker:us-west-2:123412341234:labeling-job/initial-active-learning-model', 'LabelingJobResourceConfig': {'VolumeKmsKeyId': 'string'}}", + "HumanTaskConfig": "{'WorkteamArn': 'string', 'UiConfig': {'UiTemplateS3Uri': 's3://labeling-job/ui-config.tar.gz', 'HumanTaskUiArn': 'string'}, 'PreHumanTaskLambdaArn': 'string', 'TaskKeywords': ['string'], 'TaskTitle': 'string', 'TaskDescription': 'string', 'NumberOfHumanWorkersPerDataObject': 123, 'TaskTimeLimitInSeconds': 123, 'TaskAvailabilityLifetimeInSeconds': 123, 'MaxConcurrentTaskCount': 123, 'AnnotationConsolidationConfig': {'AnnotationConsolidationLambdaArn': 'string'}, 'PublicWorkforceTaskPrice': {'AmountInUsd': {'Dollars': 123, 'Cents': 123, 'TenthFractionsOfACent': 123}}}", + "Tags": "[{'Key': 'string', 'Value': 'string'}]", + "LabelingJobOutput": "{'OutputDatasetS3Uri': 's3://labeling-job/output-dataset.tar.gz', 'FinalActiveLearningModelArn': 'arn:aws:sagemaker:us-west-2:123412341234:labeling-job/final-active-learning-model'}", + "jobType": "labeling" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/labeling-jobs/a-labeling-job", + "name": "a-labeling-job", + "type": { + "string": "SAGEMAKER" + }, + "status": "STARTING" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/labeling" + ] + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:s3,labeling-job/category-config.tar_gz,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:s3,labeling-job/data-source.tar_gz,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:s3,labeling-job/output-config.tar_gz,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:s3,labeling-job/output-dataset.tar_gz,PROD)" + ], + "inputDatajobs": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { - "urn": "urn:li:dataFlow:(sagemaker,training:a-training-job,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { - "customProperties": {}, - "externalUrl": null, - "name": "a-training-job", - "description": null, - "project": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { + "urn": "urn:li:dataFlow:(sagemaker,processing:a-processing-job,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { + "customProperties": {}, + "name": "a-processing-job" + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,training:a-training-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:training-job/a-training-job)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "customProperties": { - "TrainingJobName": "a-training-job", - "TrainingJobArn": "arn:aws:sagemaker:us-west-2:123412341234:training-job/a-training-job", - "TuningJobArn": "string", - "LabelingJobArn": "string", - "AutoMLJobArn": "string", - "ModelArtifacts": "{'S3ModelArtifacts': 's3://the-first-model-data-url/data.tar.gz'}", - "TrainingJobStatus": "InProgress", - "SecondaryStatus": "Starting", - "FailureReason": "string", - "HyperParameters": "{'parameter-1': 'some-value', 'parameter-2': 'another-value'}", - "AlgorithmSpecification": "{'TrainingImage': 'string', 'AlgorithmName': 'string', 'TrainingInputMode': 'Pipe', 'MetricDefinitions': [{'Name': 'string', 'Regex': 'string'}], 'EnableSageMakerMetricsTimeSeries': True}", - "RoleArn": "arn:aws:iam::123412341234:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole", - "InputDataConfig": "[{'ChannelName': 'string', 'DataSource': {'S3DataSource': {'S3DataType': 'ManifestFile', 'S3Uri': 's3://training-job/input-dataset.tar.gz', 'S3DataDistributionType': 'FullyReplicated', 'AttributeNames': ['string']}, 'FileSystemDataSource': {'FileSystemId': 'abcdefgihjklmnopqrstuvwxyz', 'FileSystemAccessMode': 'rw', 'FileSystemType': 'EFS', 'DirectoryPath': 'string'}}, 'ContentType': 'string', 'CompressionType': 'None', 'RecordWrapperType': 'None', 'InputMode': 'Pipe', 'ShuffleConfig': {'Seed': 123}}]", - "OutputDataConfig": "{'KmsKeyId': 'string', 'S3OutputPath': 's3://training-job/output-data.tar.gz'}", - "ResourceConfig": "{'InstanceType': 'ml.m4.xlarge', 'InstanceCount': 123, 'VolumeSizeInGB': 123, 'VolumeKmsKeyId': 'string'}", - "VpcConfig": "{'SecurityGroupIds': ['string'], 'Subnets': ['string']}", - "StoppingCondition": "{'MaxRuntimeInSeconds': 123, 'MaxWaitTimeInSeconds': 123}", - "CreationTime": "2015-01-01 00:00:00+00:00", - "TrainingStartTime": "2015-01-01 00:00:00+00:00", - "TrainingEndTime": "2015-01-01 00:00:00+00:00", - "LastModifiedTime": "2015-01-01 00:00:00+00:00", - "SecondaryStatusTransitions": "[{'Status': 'Starting', 'StartTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'EndTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'StatusMessage': 'string'}]", - "FinalMetricDataList": "[{'MetricName': 'some-metric', 'Value': 1.0, 'Timestamp': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'MetricName': 'another-metric', 'Value': 1.0, 'Timestamp': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'MetricName': 'some-metric', 'Value': 0.0, 'Timestamp': datetime.datetime(2014, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}]", - "EnableNetworkIsolation": "True", - "EnableInterContainerTrafficEncryption": "True", - "EnableManagedSpotTraining": "True", - "CheckpointConfig": "{'S3Uri': 's3://training-job/checkpoint-config.tar.gz', 'LocalPath': 'string'}", - "TrainingTimeInSeconds": "123", - "BillableTimeInSeconds": "123", - "DebugHookConfig": "{'LocalPath': 'string', 'S3OutputPath': 's3://training-job/debug-hook-config.tar.gz', 'HookParameters': {'string': 'string'}, 'CollectionConfigurations': [{'CollectionName': 'string', 'CollectionParameters': {'string': 'string'}}]}", - "ExperimentConfig": "{'ExperimentName': 'string', 'TrialName': 'string', 'TrialComponentDisplayName': 'string'}", - "DebugRuleConfigurations": "[{'RuleConfigurationName': 'string', 'LocalPath': 'string', 'S3OutputPath': 's3://training-job/debug-rule-config.tar.gz', 'RuleEvaluatorImage': 'string', 'InstanceType': 'ml.t3.medium', 'VolumeSizeInGB': 123, 'RuleParameters': {'string': 'string'}}]", - "TensorBoardOutputConfig": "{'LocalPath': 'string', 'S3OutputPath': 's3://training-job/tensorboard-output-config.tar.gz'}", - "DebugRuleEvaluationStatuses": "[{'RuleConfigurationName': 'string', 'RuleEvaluationJobArn': 'string', 'RuleEvaluationStatus': 'InProgress', 'StatusDetails': 'string', 'LastModifiedTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}]", - "ProfilerConfig": "{'S3OutputPath': 's3://training-job/profiler-config.tar.gz', 'ProfilingIntervalInMilliseconds': 123, 'ProfilingParameters': {'string': 'string'}}", - "ProfilerRuleConfigurations": "[{'RuleConfigurationName': 'string', 'LocalPath': 'string', 'S3OutputPath': 's3://training-job/profiler-rule-config.tar.gz', 'RuleEvaluatorImage': 'string', 'InstanceType': 'ml.t3.medium', 'VolumeSizeInGB': 123, 'RuleParameters': {'string': 'string'}}]", - "ProfilerRuleEvaluationStatuses": "[{'RuleConfigurationName': 'string', 'RuleEvaluationJobArn': 'string', 'RuleEvaluationStatus': 'InProgress', 'StatusDetails': 'string', 'LastModifiedTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}]", - "ProfilingStatus": "Enabled", - "RetryStrategy": "{'MaximumRetryAttempts': 123}", - "Environment": "{'string': 'string'}", - "jobType": "training" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/jobs/a-training-job", - "name": "a-training-job", - "description": null, - "type": { - "string": "SAGEMAKER" - }, - "flowUrn": null, - "status": "IN_PROGRESS" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/training" - ] - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/input-dataset.tar_gz,PROD)" - ], - "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/checkpoint-config.tar_gz,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/debug-hook-config.tar_gz,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/debug-rule-config.tar_gz,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/output-data.tar_gz,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/profiler-config.tar_gz,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/profiler-rule-config.tar_gz,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/tensorboard-output-config.tar_gz,PROD)" - ], - "inputDatajobs": [] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,processing:a-processing-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:processing-job/a-processing-job)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "ProcessingJobName": "a-processing-job", + "ProcessingJobArn": "arn:aws:sagemaker:us-west-2:123412341234:processing-job/a-processing-job", + "ProcessingInputs": "[{'InputName': 'string', 'AppManaged': True, 'S3Input': {'S3Uri': 's3://processing-job/input-data.tar.gz', 'LocalPath': 'string', 'S3DataType': 'ManifestFile', 'S3InputMode': 'Pipe', 'S3DataDistributionType': 'FullyReplicated', 'S3CompressionType': 'None'}, 'DatasetDefinition': {'AthenaDatasetDefinition': {'Catalog': 'athena-catalog', 'Database': 'athena-database', 'QueryString': 'athena-query-string', 'WorkGroup': 'athena-work-group', 'OutputS3Uri': 's3://processing-job/athena-output.tar.gz', 'KmsKeyId': 'string', 'OutputFormat': 'PARQUET', 'OutputCompression': 'GZIP'}, 'RedshiftDatasetDefinition': {'ClusterId': 'redshift-cluster', 'Database': 'redshift-database', 'DbUser': 'redshift-db-user', 'QueryString': 'redshift-query-string', 'ClusterRoleArn': 'arn:aws:sagemaker:us-west-2:123412341234:processing-job/redshift-cluster', 'OutputS3Uri': 's3://processing-job/redshift-output.tar.gz', 'KmsKeyId': 'string', 'OutputFormat': 'PARQUET', 'OutputCompression': 'None'}, 'LocalPath': 'string', 'DataDistributionType': 'FullyReplicated', 'InputMode': 'Pipe'}}]", + "ProcessingOutputConfig": "{'Outputs': [{'OutputName': 'string', 'S3Output': {'S3Uri': 's3://processing-job/processing-output.tar.gz', 'LocalPath': 'string', 'S3UploadMode': 'Continuous'}, 'FeatureStoreOutput': {'FeatureGroupName': 'string'}, 'AppManaged': True}], 'KmsKeyId': 'string'}", + "ProcessingResources": "{'ClusterConfig': {'InstanceCount': 123, 'InstanceType': 'ml.t3.medium', 'VolumeSizeInGB': 123, 'VolumeKmsKeyId': 'string'}}", + "StoppingCondition": "{'MaxRuntimeInSeconds': 123}", + "AppSpecification": "{'ImageUri': 'string', 'ContainerEntrypoint': ['string'], 'ContainerArguments': ['string']}", + "Environment": "{'string': 'string'}", + "NetworkConfig": "{'EnableInterContainerTrafficEncryption': True, 'EnableNetworkIsolation': True, 'VpcConfig': {'SecurityGroupIds': ['string'], 'Subnets': ['string']}}", + "RoleArn": "arn:aws:iam::123412341234:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole", + "ExperimentConfig": "{'ExperimentName': 'string', 'TrialName': 'string', 'TrialComponentDisplayName': 'string'}", + "ProcessingJobStatus": "InProgress", + "ExitMessage": "string", + "FailureReason": "string", + "ProcessingEndTime": "2015-01-01 00:00:00+00:00", + "ProcessingStartTime": "2015-01-01 00:00:00+00:00", + "LastModifiedTime": "2015-01-01 00:00:00+00:00", + "CreationTime": "2015-01-01 00:00:00+00:00", + "MonitoringScheduleArn": "string", + "AutoMLJobArn": "arn:aws:sagemaker:us-west-2:123412341234:auto-ml-job/an-auto-ml-job", + "TrainingJobArn": "arn:aws:sagemaker:us-west-2:123412341234:training-job/a-training-job", + "jobType": "processing" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/processing-jobs/a-processing-job", + "name": "a-processing-job", + "type": { + "string": "SAGEMAKER" + }, + "status": "IN_PROGRESS" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/processing" + ] + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:s3,processing-job/input-data.tar_gz,PROD)" + ], + "outputDatasets": [], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,auto_ml:an-auto-ml-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:auto-ml-job/an-auto-ml-job)", + "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,training:a-training-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:training-job/a-training-job)" + ] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { - "urn": "urn:li:dataFlow:(sagemaker,transform:a-transform-job,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { - "customProperties": {}, - "externalUrl": null, - "name": "a-transform-job", - "description": null, - "project": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { + "urn": "urn:li:dataFlow:(sagemaker,training:a-training-job,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { + "customProperties": {}, + "name": "a-training-job" + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,transform:a-transform-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:transform-job/a-transform-job)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "customProperties": { - "TransformJobName": "a-transform-job", - "TransformJobArn": "arn:aws:sagemaker:us-west-2:123412341234:transform-job/a-transform-job", - "TransformJobStatus": "InProgress", - "FailureReason": "string", - "ModelName": "the-second-model", - "MaxConcurrentTransforms": "123", - "ModelClientConfig": "{'InvocationsTimeoutInSeconds': 123, 'InvocationsMaxRetries': 123}", - "MaxPayloadInMB": "123", - "BatchStrategy": "MultiRecord", - "Environment": "{'string': 'string'}", - "TransformInput": "{'DataSource': {'S3DataSource': {'S3DataType': 'ManifestFile', 'S3Uri': 's3://transform-job/input-data-source.tar.gz'}}, 'ContentType': 'string', 'CompressionType': 'None', 'SplitType': 'None'}", - "TransformOutput": "{'S3OutputPath': 's3://transform-job/output.tar.gz', 'Accept': 'string', 'AssembleWith': 'None', 'KmsKeyId': 'string'}", - "TransformResources": "{'InstanceType': 'ml.m4.xlarge', 'InstanceCount': 123, 'VolumeKmsKeyId': 'string'}", - "CreationTime": "2015-01-01 00:00:00+00:00", - "TransformStartTime": "2015-01-01 00:00:00+00:00", - "TransformEndTime": "2015-01-01 00:00:00+00:00", - "LabelingJobArn": "arn:aws:sagemaker:us-west-2:123412341234:labeling-job/a-labeling-job", - "AutoMLJobArn": "arn:aws:sagemaker:us-west-2:123412341234:auto-ml-job/an-auto-ml-job", - "DataProcessing": "{'InputFilter': 'string', 'OutputFilter': 'string', 'JoinSource': 'Input'}", - "ExperimentConfig": "{'ExperimentName': 'string', 'TrialName': 'string', 'TrialComponentDisplayName': 'string'}", - "jobType": "transform" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/transform-jobs/a-transform-job", - "name": "a-transform-job", - "description": null, - "type": { - "string": "SAGEMAKER" - }, - "flowUrn": null, - "status": "IN_PROGRESS" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/transform" - ] - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:s3,transform-job/input-data-source.tar_gz,PROD)" - ], - "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:s3,transform-job/output.tar_gz,PROD)" - ], - "inputDatajobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,auto_ml:an-auto-ml-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:auto-ml-job/an-auto-ml-job)", - "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,labeling:a-labeling-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:labeling-job/a-labeling-job)" - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,training:a-training-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:training-job/a-training-job)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "TrainingJobName": "a-training-job", + "TrainingJobArn": "arn:aws:sagemaker:us-west-2:123412341234:training-job/a-training-job", + "TuningJobArn": "string", + "LabelingJobArn": "string", + "AutoMLJobArn": "string", + "ModelArtifacts": "{'S3ModelArtifacts': 's3://the-first-model-data-url/data.tar.gz'}", + "TrainingJobStatus": "InProgress", + "SecondaryStatus": "Starting", + "FailureReason": "string", + "HyperParameters": "{'parameter-1': 'some-value', 'parameter-2': 'another-value'}", + "AlgorithmSpecification": "{'TrainingImage': 'string', 'AlgorithmName': 'string', 'TrainingInputMode': 'Pipe', 'MetricDefinitions': [{'Name': 'string', 'Regex': 'string'}], 'EnableSageMakerMetricsTimeSeries': True}", + "RoleArn": "arn:aws:iam::123412341234:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole", + "InputDataConfig": "[{'ChannelName': 'string', 'DataSource': {'S3DataSource': {'S3DataType': 'ManifestFile', 'S3Uri': 's3://training-job/input-dataset.tar.gz', 'S3DataDistributionType': 'FullyReplicated', 'AttributeNames': ['string']}, 'FileSystemDataSource': {'FileSystemId': 'abcdefgihjklmnopqrstuvwxyz', 'FileSystemAccessMode': 'rw', 'FileSystemType': 'EFS', 'DirectoryPath': 'string'}}, 'ContentType': 'string', 'CompressionType': 'None', 'RecordWrapperType': 'None', 'InputMode': 'Pipe', 'ShuffleConfig': {'Seed': 123}}]", + "OutputDataConfig": "{'KmsKeyId': 'string', 'S3OutputPath': 's3://training-job/output-data.tar.gz'}", + "ResourceConfig": "{'InstanceType': 'ml.m4.xlarge', 'InstanceCount': 123, 'VolumeSizeInGB': 123, 'VolumeKmsKeyId': 'string'}", + "VpcConfig": "{'SecurityGroupIds': ['string'], 'Subnets': ['string']}", + "StoppingCondition": "{'MaxRuntimeInSeconds': 123, 'MaxWaitTimeInSeconds': 123}", + "CreationTime": "2015-01-01 00:00:00+00:00", + "TrainingStartTime": "2015-01-01 00:00:00+00:00", + "TrainingEndTime": "2015-01-01 00:00:00+00:00", + "LastModifiedTime": "2015-01-01 00:00:00+00:00", + "SecondaryStatusTransitions": "[{'Status': 'Starting', 'StartTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'EndTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), 'StatusMessage': 'string'}]", + "FinalMetricDataList": "[{'MetricName': 'some-metric', 'Value': 1.0, 'Timestamp': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'MetricName': 'another-metric', 'Value': 1.0, 'Timestamp': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'MetricName': 'some-metric', 'Value': 0.0, 'Timestamp': datetime.datetime(2014, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}]", + "EnableNetworkIsolation": "True", + "EnableInterContainerTrafficEncryption": "True", + "EnableManagedSpotTraining": "True", + "CheckpointConfig": "{'S3Uri': 's3://training-job/checkpoint-config.tar.gz', 'LocalPath': 'string'}", + "TrainingTimeInSeconds": "123", + "BillableTimeInSeconds": "123", + "DebugHookConfig": "{'LocalPath': 'string', 'S3OutputPath': 's3://training-job/debug-hook-config.tar.gz', 'HookParameters': {'string': 'string'}, 'CollectionConfigurations': [{'CollectionName': 'string', 'CollectionParameters': {'string': 'string'}}]}", + "ExperimentConfig": "{'ExperimentName': 'string', 'TrialName': 'string', 'TrialComponentDisplayName': 'string'}", + "DebugRuleConfigurations": "[{'RuleConfigurationName': 'string', 'LocalPath': 'string', 'S3OutputPath': 's3://training-job/debug-rule-config.tar.gz', 'RuleEvaluatorImage': 'string', 'InstanceType': 'ml.t3.medium', 'VolumeSizeInGB': 123, 'RuleParameters': {'string': 'string'}}]", + "TensorBoardOutputConfig": "{'LocalPath': 'string', 'S3OutputPath': 's3://training-job/tensorboard-output-config.tar.gz'}", + "DebugRuleEvaluationStatuses": "[{'RuleConfigurationName': 'string', 'RuleEvaluationJobArn': 'string', 'RuleEvaluationStatus': 'InProgress', 'StatusDetails': 'string', 'LastModifiedTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}]", + "ProfilerConfig": "{'S3OutputPath': 's3://training-job/profiler-config.tar.gz', 'ProfilingIntervalInMilliseconds': 123, 'ProfilingParameters': {'string': 'string'}}", + "ProfilerRuleConfigurations": "[{'RuleConfigurationName': 'string', 'LocalPath': 'string', 'S3OutputPath': 's3://training-job/profiler-rule-config.tar.gz', 'RuleEvaluatorImage': 'string', 'InstanceType': 'ml.t3.medium', 'VolumeSizeInGB': 123, 'RuleParameters': {'string': 'string'}}]", + "ProfilerRuleEvaluationStatuses": "[{'RuleConfigurationName': 'string', 'RuleEvaluationJobArn': 'string', 'RuleEvaluationStatus': 'InProgress', 'StatusDetails': 'string', 'LastModifiedTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}]", + "ProfilingStatus": "Enabled", + "RetryStrategy": "{'MaximumRetryAttempts': 123}", + "Environment": "{'string': 'string'}", + "jobType": "training" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/jobs/a-training-job", + "name": "a-training-job", + "type": { + "string": "SAGEMAKER" + }, + "status": "IN_PROGRESS" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/training" + ] + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/input-dataset.tar_gz,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/checkpoint-config.tar_gz,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/debug-hook-config.tar_gz,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/debug-rule-config.tar_gz,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/output-data.tar_gz,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/profiler-config.tar_gz,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/profiler-rule-config.tar_gz,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/tensorboard-output-config.tar_gz,PROD)" + ], + "inputDatajobs": [] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLModelDeploymentSnapshot": { - "urn": "urn:li:mlModelDeployment:(urn:li:dataPlatform:sagemaker,the-first-endpoint,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLModelDeploymentProperties": { - "customProperties": { - "EndpointArn": "arn:aws:sagemaker:us-west-2:123412341234:endpoint/the-first-endpoint", - "EndpointConfigName": "string", - "ProductionVariants": "[{'VariantName': 'string', 'DeployedImages': [{'SpecifiedImage': 'string', 'ResolvedImage': 'string', 'ResolutionTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], 'CurrentWeight': 0.1, 'DesiredWeight': 0.1, 'CurrentInstanceCount': 123, 'DesiredInstanceCount': 123}]", - "DataCaptureConfig": "{'EnableCapture': True, 'CaptureStatus': 'Started', 'CurrentSamplingPercentage': 123, 'DestinationS3Uri': 'string', 'KmsKeyId': 'string'}", - "EndpointStatus": "InService", - "FailureReason": "string", - "LastModifiedTime": "2015-01-01 00:00:00+00:00", - "LastDeploymentConfig": "{'BlueGreenUpdatePolicy': {'TrafficRoutingConfiguration': {'Type': 'ALL_AT_ONCE', 'WaitIntervalInSeconds': 123, 'CanarySize': {'Type': 'INSTANCE_COUNT', 'Value': 123}}, 'TerminationWaitInSeconds': 123, 'MaximumExecutionTimeoutInSeconds': 600}, 'AutoRollbackConfiguration': {'Alarms': [{'AlarmName': 'string'}]}}" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/endpoints/the-first-endpoint", - "description": null, - "createdAt": 1420070400000, - "version": null, - "status": "IN_SERVICE" - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { + "urn": "urn:li:dataFlow:(sagemaker,transform:a-transform-job,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { + "customProperties": {}, + "name": "a-transform-job" + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLModelDeploymentSnapshot": { - "urn": "urn:li:mlModelDeployment:(urn:li:dataPlatform:sagemaker,the-second-endpoint,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLModelDeploymentProperties": { - "customProperties": { - "EndpointArn": "arn:aws:sagemaker:us-west-2:123412341234:endpoint/the-second-endpoint", - "EndpointConfigName": "string", - "ProductionVariants": "[{'VariantName': 'string', 'DeployedImages': [{'SpecifiedImage': 'string', 'ResolvedImage': 'string', 'ResolutionTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], 'CurrentWeight': 0.1, 'DesiredWeight': 0.1, 'CurrentInstanceCount': 123, 'DesiredInstanceCount': 123}]", - "DataCaptureConfig": "{'EnableCapture': True, 'CaptureStatus': 'Started', 'CurrentSamplingPercentage': 123, 'DestinationS3Uri': 'string', 'KmsKeyId': 'string'}", - "EndpointStatus": "Creating", - "FailureReason": "string", - "LastModifiedTime": "2015-01-01 00:00:00+00:00", - "LastDeploymentConfig": "{'BlueGreenUpdatePolicy': {'TrafficRoutingConfiguration': {'Type': 'ALL_AT_ONCE', 'WaitIntervalInSeconds': 123, 'CanarySize': {'Type': 'INSTANCE_COUNT', 'Value': 123}}, 'TerminationWaitInSeconds': 123, 'MaximumExecutionTimeoutInSeconds': 600}, 'AutoRollbackConfiguration': {'Alarms': [{'AlarmName': 'string'}]}}" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/endpoints/the-second-endpoint", - "description": null, - "createdAt": 1420070400000, - "version": null, - "status": "CREATING" - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,transform:a-transform-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:transform-job/a-transform-job)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "TransformJobName": "a-transform-job", + "TransformJobArn": "arn:aws:sagemaker:us-west-2:123412341234:transform-job/a-transform-job", + "TransformJobStatus": "InProgress", + "FailureReason": "string", + "ModelName": "the-second-model", + "MaxConcurrentTransforms": "123", + "ModelClientConfig": "{'InvocationsTimeoutInSeconds': 123, 'InvocationsMaxRetries': 123}", + "MaxPayloadInMB": "123", + "BatchStrategy": "MultiRecord", + "Environment": "{'string': 'string'}", + "TransformInput": "{'DataSource': {'S3DataSource': {'S3DataType': 'ManifestFile', 'S3Uri': 's3://transform-job/input-data-source.tar.gz'}}, 'ContentType': 'string', 'CompressionType': 'None', 'SplitType': 'None'}", + "TransformOutput": "{'S3OutputPath': 's3://transform-job/output.tar.gz', 'Accept': 'string', 'AssembleWith': 'None', 'KmsKeyId': 'string'}", + "TransformResources": "{'InstanceType': 'ml.m4.xlarge', 'InstanceCount': 123, 'VolumeKmsKeyId': 'string'}", + "CreationTime": "2015-01-01 00:00:00+00:00", + "TransformStartTime": "2015-01-01 00:00:00+00:00", + "TransformEndTime": "2015-01-01 00:00:00+00:00", + "LabelingJobArn": "arn:aws:sagemaker:us-west-2:123412341234:labeling-job/a-labeling-job", + "AutoMLJobArn": "arn:aws:sagemaker:us-west-2:123412341234:auto-ml-job/an-auto-ml-job", + "DataProcessing": "{'InputFilter': 'string', 'OutputFilter': 'string', 'JoinSource': 'Input'}", + "ExperimentConfig": "{'ExperimentName': 'string', 'TrialName': 'string', 'TrialComponentDisplayName': 'string'}", + "jobType": "transform" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/transform-jobs/a-transform-job", + "name": "a-transform-job", + "type": { + "string": "SAGEMAKER" + }, + "status": "IN_PROGRESS" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/transform" + ] + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:s3,transform-job/input-data-source.tar_gz,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:s3,transform-job/output.tar_gz,PROD)" + ], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,auto_ml:an-auto-ml-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:auto-ml-job/an-auto-ml-job)", + "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,labeling:a-labeling-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:labeling-job/a-labeling-job)" + ] + } + } + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLModelGroupSnapshot": { - "urn": "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,a-model-package-group,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLModelGroupProperties": { - "customProperties": { - "ModelPackageGroupArn": "arn:aws:sagemaker:us-west-2:123412341234:model-package-group/a-model-package-group", - "ModelPackageGroupDescription": "Just a model package group.", - "CreatedBy": "{'UserProfileArn': 'arn:aws:sagemaker:us-west-2:123412341234:user-profile/some-domain/some-user', 'UserProfileName': 'some-user', 'DomainId': 'some-domain'}", - "ModelPackageGroupStatus": "Completed" - }, - "description": "Just a model package group.", - "createdAt": 1420070400000, - "version": null - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:some-user", - "type": "DATAOWNER", - "source": null + "com.linkedin.pegasus2avro.metadata.snapshot.MLModelDeploymentSnapshot": { + "urn": "urn:li:mlModelDeployment:(urn:li:dataPlatform:sagemaker,the-first-endpoint,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLModelDeploymentProperties": { + "customProperties": { + "EndpointArn": "arn:aws:sagemaker:us-west-2:123412341234:endpoint/the-first-endpoint", + "EndpointConfigName": "string", + "ProductionVariants": "[{'VariantName': 'string', 'DeployedImages': [{'SpecifiedImage': 'string', 'ResolvedImage': 'string', 'ResolutionTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], 'CurrentWeight': 0.1, 'DesiredWeight': 0.1, 'CurrentInstanceCount': 123, 'DesiredInstanceCount': 123}]", + "DataCaptureConfig": "{'EnableCapture': True, 'CaptureStatus': 'Started', 'CurrentSamplingPercentage': 123, 'DestinationS3Uri': 'string', 'KmsKeyId': 'string'}", + "EndpointStatus": "InService", + "FailureReason": "string", + "LastModifiedTime": "2015-01-01 00:00:00+00:00", + "LastDeploymentConfig": "{'BlueGreenUpdatePolicy': {'TrafficRoutingConfiguration': {'Type': 'ALL_AT_ONCE', 'WaitIntervalInSeconds': 123, 'CanarySize': {'Type': 'INSTANCE_COUNT', 'Value': 123}}, 'TerminationWaitInSeconds': 123, 'MaximumExecutionTimeoutInSeconds': 600}, 'AutoRollbackConfiguration': {'Alarms': [{'AlarmName': 'string'}]}}" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/endpoints/the-first-endpoint", + "createdAt": 1420070400000, + "status": "IN_SERVICE" + } } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/sagemaker" - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLModelSnapshot": { - "urn": "urn:li:mlModel:(urn:li:dataPlatform:sagemaker,the-first-model,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLModelProperties": { - "customProperties": { - "PrimaryContainer": "{'ContainerHostname': 'string', 'Image': '123412341234.dkr.ecr.us-west-2.amazonaws.com/the-first-model-image', 'ImageConfig': {'RepositoryAccessMode': 'Platform', 'RepositoryAuthConfig': {'RepositoryCredentialsProviderArn': 'string'}}, 'Mode': 'SingleModel', 'ModelDataUrl': 's3://the-first-model-data-url/data.tar.gz', 'Environment': {'string': 'string'}, 'ModelPackageName': 'string', 'MultiModelConfig': {'ModelCacheSetting': 'Enabled'}}", - "Containers": "[{'ContainerHostname': 'string', 'Image': 'string', 'ImageConfig': {'RepositoryAccessMode': 'Platform', 'RepositoryAuthConfig': {'RepositoryCredentialsProviderArn': 'string'}}, 'Mode': 'SingleModel', 'ModelDataUrl': 's3://training-job-2/model-artifact.tar.gz', 'Environment': {'string': 'string'}, 'ModelPackageName': 'string', 'MultiModelConfig': {'ModelCacheSetting': 'Enabled'}}]", - "InferenceExecutionConfig": "{'Mode': 'Serial'}", - "ExecutionRoleArn": "arn:aws:iam::123412341234:role/service-role/AmazonSageMaker-ExecutionRole-20210614T104201", - "VpcConfig": "{'SecurityGroupIds': ['string'], 'Subnets': ['string']}", - "ModelArn": "arn:aws:sagemaker:us-west-2:123412341234:model/the-first-model", - "EnableNetworkIsolation": "True" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/models/the-first-model", - "description": null, - "date": 1420070400000, - "version": null, - "type": null, - "hyperParameters": null, - "hyperParams": [ - { - "name": "parameter-1", - "description": null, - "value": "some-value", - "createdAt": null - }, + "com.linkedin.pegasus2avro.metadata.snapshot.MLModelDeploymentSnapshot": { + "urn": "urn:li:mlModelDeployment:(urn:li:dataPlatform:sagemaker,the-second-endpoint,PROD)", + "aspects": [ { - "name": "parameter-2", - "description": null, - "value": "another-value", - "createdAt": null + "com.linkedin.pegasus2avro.ml.metadata.MLModelDeploymentProperties": { + "customProperties": { + "EndpointArn": "arn:aws:sagemaker:us-west-2:123412341234:endpoint/the-second-endpoint", + "EndpointConfigName": "string", + "ProductionVariants": "[{'VariantName': 'string', 'DeployedImages': [{'SpecifiedImage': 'string', 'ResolvedImage': 'string', 'ResolutionTime': datetime.datetime(2015, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], 'CurrentWeight': 0.1, 'DesiredWeight': 0.1, 'CurrentInstanceCount': 123, 'DesiredInstanceCount': 123}]", + "DataCaptureConfig": "{'EnableCapture': True, 'CaptureStatus': 'Started', 'CurrentSamplingPercentage': 123, 'DestinationS3Uri': 'string', 'KmsKeyId': 'string'}", + "EndpointStatus": "Creating", + "FailureReason": "string", + "LastModifiedTime": "2015-01-01 00:00:00+00:00", + "LastDeploymentConfig": "{'BlueGreenUpdatePolicy': {'TrafficRoutingConfiguration': {'Type': 'ALL_AT_ONCE', 'WaitIntervalInSeconds': 123, 'CanarySize': {'Type': 'INSTANCE_COUNT', 'Value': 123}}, 'TerminationWaitInSeconds': 123, 'MaximumExecutionTimeoutInSeconds': 600}, 'AutoRollbackConfiguration': {'Alarms': [{'AlarmName': 'string'}]}}" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/endpoints/the-second-endpoint", + "createdAt": 1420070400000, + "status": "CREATING" + } } - ], - "trainingMetrics": [ + ] + } + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLModelGroupSnapshot": { + "urn": "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,a-model-package-group,PROD)", + "aspects": [ { - "name": "another-metric", - "description": null, - "value": "1.0", - "createdAt": null + "com.linkedin.pegasus2avro.ml.metadata.MLModelGroupProperties": { + "customProperties": { + "ModelPackageGroupArn": "arn:aws:sagemaker:us-west-2:123412341234:model-package-group/a-model-package-group", + "ModelPackageGroupDescription": "Just a model package group.", + "CreatedBy": "{'UserProfileArn': 'arn:aws:sagemaker:us-west-2:123412341234:user-profile/some-domain/some-user', 'UserProfileName': 'some-user', 'DomainId': 'some-domain'}", + "ModelPackageGroupStatus": "Completed" + }, + "description": "Just a model package group.", + "createdAt": 1420070400000 + } }, { - "name": "some-metric", - "description": null, - "value": "1.0", - "createdAt": null + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:some-user", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/sagemaker" + ] + } } - ], - "onlineMetrics": null, - "mlFeatures": null, - "tags": [], - "deployments": [ - "urn:li:mlModelDeployment:(urn:li:dataPlatform:sagemaker,arn:aws:sagemaker:us-west-2:123412341234:endpoint/the-first-endpoint,PROD)" - ], - "trainingJobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,training:a-training-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:training-job/a-training-job)" - ], - "downstreamJobs": [], - "groups": [ - "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,a-model-package-group,PROD)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/sagemaker/a-model-package-group" - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - }, - { - "auditHeader": null, + ] + } + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLModelSnapshot": { - "urn": "urn:li:mlModel:(urn:li:dataPlatform:sagemaker,the-second-model,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.ml.metadata.MLModelProperties": { - "customProperties": { - "PrimaryContainer": "{'ContainerHostname': 'string', 'Image': '123412341234.dkr.ecr.us-west-2.amazonaws.com/the-second-model-image', 'ImageConfig': {'RepositoryAccessMode': 'Platform', 'RepositoryAuthConfig': {'RepositoryCredentialsProviderArn': 'string'}}, 'Mode': 'MultiModel', 'ModelDataUrl': 's3://the-second-model-data-url/data.tar.gz', 'Environment': {'string': 'string'}, 'ModelPackageName': 'string', 'MultiModelConfig': {'ModelCacheSetting': 'Disabled'}}", - "Containers": "[{'ContainerHostname': 'string', 'Image': 'string', 'ImageConfig': {'RepositoryAccessMode': 'Vpc', 'RepositoryAuthConfig': {'RepositoryCredentialsProviderArn': 'string'}}, 'Mode': 'SingleModel', 'ModelDataUrl': 's3://the-first-model-data-url/data.tar.gz', 'Environment': {'string': 'string'}, 'ModelPackageName': 'string', 'MultiModelConfig': {'ModelCacheSetting': 'Disabled'}}]", - "InferenceExecutionConfig": "{'Mode': 'Serial'}", - "ExecutionRoleArn": "arn:aws:iam::123412341234:role/service-role/AmazonSageMaker-ExecutionRole-20210614T104201", - "VpcConfig": "{'SecurityGroupIds': ['string'], 'Subnets': ['string']}", - "ModelArn": "arn:aws:sagemaker:us-west-2:123412341234:model/the-second-model", - "EnableNetworkIsolation": "False" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/models/the-second-model", - "description": null, - "date": 1420070400000, - "version": null, - "type": null, - "hyperParameters": null, - "hyperParams": [ - { - "name": "parameter-1", - "description": null, - "value": "some-value", - "createdAt": null + "com.linkedin.pegasus2avro.metadata.snapshot.MLModelSnapshot": { + "urn": "urn:li:mlModel:(urn:li:dataPlatform:sagemaker,the-first-model,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.ml.metadata.MLModelProperties": { + "customProperties": { + "PrimaryContainer": "{'ContainerHostname': 'string', 'Image': '123412341234.dkr.ecr.us-west-2.amazonaws.com/the-first-model-image', 'ImageConfig': {'RepositoryAccessMode': 'Platform', 'RepositoryAuthConfig': {'RepositoryCredentialsProviderArn': 'string'}}, 'Mode': 'SingleModel', 'ModelDataUrl': 's3://the-first-model-data-url/data.tar.gz', 'Environment': {'string': 'string'}, 'ModelPackageName': 'string', 'MultiModelConfig': {'ModelCacheSetting': 'Enabled'}}", + "Containers": "[{'ContainerHostname': 'string', 'Image': 'string', 'ImageConfig': {'RepositoryAccessMode': 'Platform', 'RepositoryAuthConfig': {'RepositoryCredentialsProviderArn': 'string'}}, 'Mode': 'SingleModel', 'ModelDataUrl': 's3://training-job-2/model-artifact.tar.gz', 'Environment': {'string': 'string'}, 'ModelPackageName': 'string', 'MultiModelConfig': {'ModelCacheSetting': 'Enabled'}}]", + "InferenceExecutionConfig": "{'Mode': 'Serial'}", + "ExecutionRoleArn": "arn:aws:iam::123412341234:role/service-role/AmazonSageMaker-ExecutionRole-20210614T104201", + "VpcConfig": "{'SecurityGroupIds': ['string'], 'Subnets': ['string']}", + "ModelArn": "arn:aws:sagemaker:us-west-2:123412341234:model/the-first-model", + "EnableNetworkIsolation": "True" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/models/the-first-model", + "date": 1420070400000, + "hyperParams": [ + { + "name": "parameter-1", + "value": "some-value" + }, + { + "name": "parameter-2", + "value": "another-value" + } + ], + "trainingMetrics": [ + { + "name": "another-metric", + "value": "1.0" + }, + { + "name": "some-metric", + "value": "1.0" + } + ], + "tags": [], + "deployments": [ + "urn:li:mlModelDeployment:(urn:li:dataPlatform:sagemaker,arn:aws:sagemaker:us-west-2:123412341234:endpoint/the-first-endpoint,PROD)" + ], + "trainingJobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,training:a-training-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:training-job/a-training-job)" + ], + "downstreamJobs": [], + "groups": [ + "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,a-model-package-group,PROD)" + ] + } }, { - "name": "parameter-2", - "description": null, - "value": "another-value", - "createdAt": null + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/sagemaker/a-model-package-group" + ] + } } - ], - "trainingMetrics": [ + ] + } + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLModelSnapshot": { + "urn": "urn:li:mlModel:(urn:li:dataPlatform:sagemaker,the-second-model,PROD)", + "aspects": [ { - "name": "another-metric", - "description": null, - "value": "1.0", - "createdAt": null + "com.linkedin.pegasus2avro.ml.metadata.MLModelProperties": { + "customProperties": { + "PrimaryContainer": "{'ContainerHostname': 'string', 'Image': '123412341234.dkr.ecr.us-west-2.amazonaws.com/the-second-model-image', 'ImageConfig': {'RepositoryAccessMode': 'Platform', 'RepositoryAuthConfig': {'RepositoryCredentialsProviderArn': 'string'}}, 'Mode': 'MultiModel', 'ModelDataUrl': 's3://the-second-model-data-url/data.tar.gz', 'Environment': {'string': 'string'}, 'ModelPackageName': 'string', 'MultiModelConfig': {'ModelCacheSetting': 'Disabled'}}", + "Containers": "[{'ContainerHostname': 'string', 'Image': 'string', 'ImageConfig': {'RepositoryAccessMode': 'Vpc', 'RepositoryAuthConfig': {'RepositoryCredentialsProviderArn': 'string'}}, 'Mode': 'SingleModel', 'ModelDataUrl': 's3://the-first-model-data-url/data.tar.gz', 'Environment': {'string': 'string'}, 'ModelPackageName': 'string', 'MultiModelConfig': {'ModelCacheSetting': 'Disabled'}}]", + "InferenceExecutionConfig": "{'Mode': 'Serial'}", + "ExecutionRoleArn": "arn:aws:iam::123412341234:role/service-role/AmazonSageMaker-ExecutionRole-20210614T104201", + "VpcConfig": "{'SecurityGroupIds': ['string'], 'Subnets': ['string']}", + "ModelArn": "arn:aws:sagemaker:us-west-2:123412341234:model/the-second-model", + "EnableNetworkIsolation": "False" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/sagemaker/home?region=us-west-2#/models/the-second-model", + "date": 1420070400000, + "hyperParams": [ + { + "name": "parameter-1", + "value": "some-value" + }, + { + "name": "parameter-2", + "value": "another-value" + } + ], + "trainingMetrics": [ + { + "name": "another-metric", + "value": "1.0" + }, + { + "name": "some-metric", + "value": "1.0" + } + ], + "tags": [], + "deployments": [ + "urn:li:mlModelDeployment:(urn:li:dataPlatform:sagemaker,arn:aws:sagemaker:us-west-2:123412341234:endpoint/the-second-endpoint,PROD)" + ], + "trainingJobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,training:a-training-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:training-job/a-training-job)" + ], + "downstreamJobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,edge_packaging:an-edge-packaging-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:edge-packaging-job/an-edge-packaging-job)", + "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,transform:a-transform-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:transform-job/a-transform-job)" + ], + "groups": [ + "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,a-model-package-group,PROD)" + ] + } }, { - "name": "some-metric", - "description": null, - "value": "1.0", - "createdAt": null + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/sagemaker/a-model-package-group" + ] + } } - ], - "onlineMetrics": null, - "mlFeatures": null, - "tags": [], - "deployments": [ - "urn:li:mlModelDeployment:(urn:li:dataPlatform:sagemaker,arn:aws:sagemaker:us-west-2:123412341234:endpoint/the-second-endpoint,PROD)" - ], - "trainingJobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,training:a-training-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:training-job/a-training-job)" - ], - "downstreamJobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,edge_packaging:an-edge-packaging-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:edge-packaging-job/an-edge-packaging-job)", - "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,transform:a-transform-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:transform-job/a-transform-job)" - ], - "groups": [ - "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,a-model-package-group,PROD)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/sagemaker/a-model-package-group" - ] - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": null - } -] + ] + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(sagemaker,auto_ml:an-auto-ml-job,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(sagemaker,compilation:a-compilation-job,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(sagemaker,edge_packaging:an-edge-packaging-job,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(sagemaker,hyper_parameter_tuning:a-hyper-parameter-tuning-job,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(sagemaker,labeling:a-labeling-job,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(sagemaker,processing:a-processing-job,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(sagemaker,training:a-training-job,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(sagemaker,transform:a-transform-job,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,auto_ml:an-auto-ml-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:auto-ml-job/an-auto-ml-job)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,compilation:a-compilation-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:compilation-job/a-compilation-job)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,edge_packaging:an-edge-packaging-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:edge-packaging-job/an-edge-packaging-job)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,hyper_parameter_tuning:a-hyper-parameter-tuning-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:hyper-parameter-tuning-job/a-hyper-parameter-tuning-job)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,labeling:a-labeling-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:labeling-job/a-labeling-job)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,processing:a-processing-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:processing-job/a-processing-job)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,training:a-training-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:training-job/a-training-job)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(sagemaker,transform:a-transform-job,PROD),arn:aws:sagemaker:us-west-2:123412341234:transform-job/a-transform-job)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,auto-ml-job-input-bucket/file_txt,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,auto-ml-job-output-bucket/file_txt,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,compilation-job-bucket/input-config.tar_gz,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,compilation-job-bucket/output-config.tar_gz,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,edge-packaging-bucket/model-artifact.tar_gz,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,edge-packaging-bucket/output-config.tar_gz,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,labeling-job/category-config.tar_gz,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,labeling-job/data-source.tar_gz,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,labeling-job/output-config.tar_gz,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,labeling-job/output-dataset.tar_gz,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,processing-job/input-data.tar_gz,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/checkpoint-config.tar_gz,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/debug-hook-config.tar_gz,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/debug-rule-config.tar_gz,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/input-dataset.tar_gz,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/output-data.tar_gz,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/profiler-config.tar_gz,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/profiler-rule-config.tar_gz,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,training-job/tensorboard-output-config.tar_gz,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,transform-job/input-data-source.tar_gz,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,transform-job/output.tar_gz,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "mlFeature", + "entityUrn": "urn:li:mlFeature:(test,feature_2)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "mlFeature", + "entityUrn": "urn:li:mlFeature:(test,feature_3)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "mlFeature", + "entityUrn": "urn:li:mlFeature:(test-1,height)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "mlFeature", + "entityUrn": "urn:li:mlFeature:(test-1,name)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "mlFeature", + "entityUrn": "urn:li:mlFeature:(test-1,time)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "mlFeature", + "entityUrn": "urn:li:mlFeature:(test-2,some-feature-1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "mlFeature", + "entityUrn": "urn:li:mlFeature:(test-2,some-feature-3)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "mlFeatureTable", + "entityUrn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:sagemaker,test)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "mlFeatureTable", + "entityUrn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:sagemaker,test-1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "mlFeatureTable", + "entityUrn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:sagemaker,test-2)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "mlModel", + "entityUrn": "urn:li:mlModel:(urn:li:dataPlatform:sagemaker,the-first-model,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "mlModel", + "entityUrn": "urn:li:mlModel:(urn:li:dataPlatform:sagemaker,the-second-model,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "mlModelDeployment", + "entityUrn": "urn:li:mlModelDeployment:(urn:li:dataPlatform:sagemaker,the-first-endpoint,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "mlModelDeployment", + "entityUrn": "urn:li:mlModelDeployment:(urn:li:dataPlatform:sagemaker,the-second-endpoint,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "mlModelGroup", + "entityUrn": "urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,a-model-package-group,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "mlPrimaryKey", + "entityUrn": "urn:li:mlPrimaryKey:(test,feature_1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "mlPrimaryKey", + "entityUrn": "urn:li:mlPrimaryKey:(test-1,id)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "mlPrimaryKey", + "entityUrn": "urn:li:mlPrimaryKey:(test-2,some-feature-2)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +} +] \ No newline at end of file From cc46729137c654161113a86c505f9e2a0e0e9ff1 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Tue, 25 Jul 2023 13:53:16 +0530 Subject: [PATCH 178/222] chore(ingest): add example of training metric/hyper parameters (#8491) --- metadata-ingestion/examples/library/create_mlmodel.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/metadata-ingestion/examples/library/create_mlmodel.py b/metadata-ingestion/examples/library/create_mlmodel.py index c324f8a820639..630e682eff842 100644 --- a/metadata-ingestion/examples/library/create_mlmodel.py +++ b/metadata-ingestion/examples/library/create_mlmodel.py @@ -31,6 +31,16 @@ description="my feature", groups=model_group_urns, mlFeatures=feature_urns, + trainingMetrics=[ + models.MLMetricClass( + name="accuracy", description="accuracy of the model", value="1.0" + ) + ], + hyperParams=[ + models.MLHyperParamClass( + name="hyper_1", description="hyper_1", value="0.102" + ) + ], ), ) From eac003ccf4aeddf96b66a5401ba48d1309ad0541 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 25 Jul 2023 01:46:27 -0700 Subject: [PATCH 179/222] feat(ingest): enable pipeline reporting by default (#8472) --- docs/how/updating-datahub.md | 1 + metadata-ingestion/src/datahub/cli/check_cli.py | 3 ++- metadata-ingestion/src/datahub/cli/docker_cli.py | 2 +- metadata-ingestion/src/datahub/cli/ingest_cli.py | 2 +- .../src/datahub/ingestion/graph/client.py | 12 ++++++------ .../src/datahub/ingestion/run/pipeline.py | 2 +- .../ingestion/transformer/add_dataset_terms.py | 5 +++-- 7 files changed, 15 insertions(+), 12 deletions(-) diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index b705c973cdbb5..ad12aacd00339 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -15,6 +15,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe certain column-level metrics. Instead, set `profile_table_level_only` to `false` and individually enable / disable desired field metrics. - #8451: The `bigquery-beta` and `snowflake-beta` source aliases have been dropped. Use `bigquery` and `snowflake` as the source type instead. +- #8472: Ingestion runs created with Pipeline.create will show up in the DataHub ingestion tab as CLI-based runs. To revert to the previous behavior of not showing these runs in DataHub, pass `no_default_report=True`. ### Potential Downtime diff --git a/metadata-ingestion/src/datahub/cli/check_cli.py b/metadata-ingestion/src/datahub/cli/check_cli.py index bec1672264b88..f20272ecd9dbf 100644 --- a/metadata-ingestion/src/datahub/cli/check_cli.py +++ b/metadata-ingestion/src/datahub/cli/check_cli.py @@ -61,7 +61,8 @@ def metadata_file(json_file: str, rewrite: bool, unpack_mces: bool) -> None: "type": "file", "config": {"filename": out_file.name}, }, - } + }, + no_default_report=True, ) pipeline.run() diff --git a/metadata-ingestion/src/datahub/cli/docker_cli.py b/metadata-ingestion/src/datahub/cli/docker_cli.py index e2b7b2a2e1ff4..918f610ce4635 100644 --- a/metadata-ingestion/src/datahub/cli/docker_cli.py +++ b/metadata-ingestion/src/datahub/cli/docker_cli.py @@ -985,7 +985,7 @@ def ingest_sample_data(path: Optional[str], token: Optional[str]) -> None: if token is not None: recipe["sink"]["config"]["token"] = token - pipeline = Pipeline.create(recipe) + pipeline = Pipeline.create(recipe, no_default_report=True) pipeline.run() ret = pipeline.pretty_print_summary() sys.exit(ret) diff --git a/metadata-ingestion/src/datahub/cli/ingest_cli.py b/metadata-ingestion/src/datahub/cli/ingest_cli.py index c8c352d1f83ff..72c15e92257aa 100644 --- a/metadata-ingestion/src/datahub/cli/ingest_cli.py +++ b/metadata-ingestion/src/datahub/cli/ingest_cli.py @@ -253,7 +253,7 @@ def mcps(path: str) -> None: }, } - pipeline = Pipeline.create(recipe) + pipeline = Pipeline.create(recipe, no_default_report=True) pipeline.run() ret = pipeline.pretty_print_summary() sys.exit(ret) diff --git a/metadata-ingestion/src/datahub/ingestion/graph/client.py b/metadata-ingestion/src/datahub/ingestion/graph/client.py index cac53c350f2ea..de8b28d4b95a8 100644 --- a/metadata-ingestion/src/datahub/ingestion/graph/client.py +++ b/metadata-ingestion/src/datahub/ingestion/graph/client.py @@ -57,12 +57,12 @@ class DatahubClientConfig(ConfigModel): """Configuration class for holding connectivity to datahub gms""" server: str = "http://localhost:8080" - token: Optional[str] - timeout_sec: Optional[int] - retry_status_codes: Optional[List[int]] - retry_max_times: Optional[int] - extra_headers: Optional[Dict[str, str]] - ca_certificate_path: Optional[str] + token: Optional[str] = None + timeout_sec: Optional[int] = None + retry_status_codes: Optional[List[int]] = None + retry_max_times: Optional[int] = None + extra_headers: Optional[Dict[str, str]] = None + ca_certificate_path: Optional[str] = None disable_ssl_verification: bool = False _max_threads_moved_to_sink = pydantic_removed_field( diff --git a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py index 7fe39ef3e64c6..79d959965e0dd 100644 --- a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py +++ b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py @@ -328,7 +328,7 @@ def create( dry_run: bool = False, preview_mode: bool = False, preview_workunits: int = 10, - report_to: Optional[str] = None, + report_to: Optional[str] = "datahub", no_default_report: bool = False, raw_config: Optional[dict] = None, ) -> "Pipeline": diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_terms.py b/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_terms.py index 996846e8dd061..f21e3ec319349 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_terms.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_terms.py @@ -132,8 +132,9 @@ class PatternAddDatasetTerms(AddDatasetTerms): def __init__(self, config: PatternDatasetTermsConfig, ctx: PipelineContext): term_pattern = config.term_pattern generic_config = AddDatasetTermsConfig( - get_terms_to_add=lambda _: [ - GlossaryTermAssociationClass(urn=urn) for urn in term_pattern.value(_) + get_terms_to_add=lambda entity_urn: [ + GlossaryTermAssociationClass(urn=term_urn) + for term_urn in term_pattern.value(entity_urn) ], replace_existing=config.replace_existing, semantics=config.semantics, From 77ace587f743df688ddb464bbe35d30a9e730d76 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Tue, 25 Jul 2023 10:02:02 -0400 Subject: [PATCH 180/222] feat(docs) Add guide for generating browsePathsV2 aspects (#8448) --- docs-website/sidebars.js | 1 + docs/browseV2/browse-paths-v2.md | 51 ++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 docs/browseV2/browse-paths-v2.md diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index b5ffd1964d7c1..7d651fd5d1894 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -486,6 +486,7 @@ module.exports = { "docs/how/add-custom-ingestion-source", "docs/how/add-custom-data-platform", "docs/advanced/browse-paths-upgrade", + "docs/browseV2/browse-paths-v2", ], }, ], diff --git a/docs/browseV2/browse-paths-v2.md b/docs/browseV2/browse-paths-v2.md new file mode 100644 index 0000000000000..b1f63b4a182ea --- /dev/null +++ b/docs/browseV2/browse-paths-v2.md @@ -0,0 +1,51 @@ +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Generating Browse Paths (V2) + + + +## Introduction + +Browse (V2) is a way for users to explore and dive deeper into their data. Its integration with the search experience allows users to combine search queries and filters with entity type and platform nested folders. + +Most entities should have a browse path that allows users to navigate the left side panel on the search page to find groups of entities under different folders that come from these browse paths. Below, you can see an example of the sidebar with some new browse paths. + +

+ +

+ +This new browse sidebar always starts with Entity Type, then optionally shows Environment (PROD, DEV, etc.) if there are 2 or more Environments, then Platform. Below the Platform level, we render out folders that come directly from entity's [browsePathsV2](https://datahubproject.io/docs/generated/metamodel/entities/dataset#browsepathsv2) aspects. + +## Generating Custom Browse Paths + +A `browsePathsV2` aspect has a field called `path` which contains a list of `BrowsePathEntry` objects. Each object in the path represents one level of the entity's browse path where the first entry is the highest level and the last entry is the lowest level. + +If an entity has this aspect filled out, their browse path will show up in the browse sidebar so that you can navigate its folders and select one to filter search results down. + +For example, in the browse sidebar on the left of the image above, there are 10 Dataset entities from the BigQuery Platform that have `browsePathsV2` aspects that look like the following: + +``` +[ { id: "bigquery-public-data" }, { id: "covid19_public_forecasts" } ] +``` + +The `id` in a `BrowsePathEntry` is required and is what will be shown in the UI unless the optional `urn` field is populated. If the `urn` field is populated, we will try to resolve this path entry into an entity object and display that entity's name. We will also show a link to allow you to open up the entity profile. + +The `urn` field should only be populated if there is an entity in your DataHub instance that belongs in that entity's browse path. This makes most sense for Datasets to have Container entities in the browse paths as well as some other cases such as a DataFlow being part of a DataJob's browse path. For any other situation, feel free to leave `urn` empty and populate `id` with the text you want to be shown in the UI for your entity's path. + +## Additional Resources + +### GraphQL + +* [browseV2](../../graphql/queries.md#browsev2) + +## FAQ and Troubleshooting + +**How are browsePathsV2 aspects created?** + +We create `browsePathsV2` aspects for all entities that should have one by default when you ingest your data if this aspect is not already provided. This happens based on separator characters that appear within an Urn. + +Our ingestion sources are also producing `browsePathsV2` aspects since CLI version v0.10.5. + +### Related Features + +* [Search](../how/search.md) From db42998a69ba1c0168b8025c5964b69ea290dfde Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Tue, 25 Jul 2023 21:58:44 +0530 Subject: [PATCH 181/222] fix(browsepathv2): default browse path with empty space (#8503) --- .../linkedin/metadata/search/utils/BrowsePathV2Utils.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/BrowsePathV2Utils.java b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/BrowsePathV2Utils.java index 1487ac58d6a0a..a7f5ea7a51e29 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/BrowsePathV2Utils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/BrowsePathV2Utils.java @@ -24,6 +24,7 @@ import java.util.Collections; import java.util.List; import java.util.regex.Pattern; +import java.util.stream.Collectors; import static com.linkedin.metadata.Constants.CONTAINER_ASPECT_NAME; @@ -140,7 +141,9 @@ private static BrowsePathEntryArray getContainerPathEntries(@Nonnull final Urn e private static BrowsePathEntryArray getDefaultDatasetPathEntries(@Nonnull final String datasetName, @Nonnull final Character delimiter) { BrowsePathEntryArray browsePathEntries = new BrowsePathEntryArray(); if (datasetName.contains(delimiter.toString())) { - final List datasetNamePathParts = Arrays.asList(datasetName.split(Pattern.quote(delimiter.toString()))); + final List datasetNamePathParts = Arrays.stream(datasetName.split(Pattern.quote(delimiter.toString()))) + .filter((name) -> !name.isEmpty()) + .collect(Collectors.toList()); // Omit the name from the path. datasetNamePathParts.subList(0, datasetNamePathParts.size() - 1).forEach((part -> { browsePathEntries.add(createBrowsePathEntry(part, null)); From b12de099aa72758faaf01b73d38250d323951a9d Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 25 Jul 2023 10:48:37 -0700 Subject: [PATCH 182/222] docs: add docs on sqlglot lineage (#8482) --- .../src/datahub/utilities/sqlglot_lineage.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py index 123a965e5a80a..57f93f27e9147 100644 --- a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py +++ b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py @@ -761,6 +761,54 @@ def sqlglot_lineage( default_db: Optional[str] = None, default_schema: Optional[str] = None, ) -> SqlParsingResult: + """Parse a SQL statement and generate lineage information. + + This is a schema-aware lineage generator, meaning that it will use the + schema information for the tables involved to generate lineage information + for the columns involved. The schema_resolver is responsible for providing + the table schema information. + + The parser supports most types of DML statements (SELECT, INSERT, UPDATE, + DELETE, MERGE) as well as CREATE TABLE AS SELECT (CTAS) statements. It + does not support DDL statements (CREATE TABLE, ALTER TABLE, etc.). + + The table-level lineage tends to be fairly reliable, while column-level + can be brittle with respect to missing schema information and complex + SQL logic like UNNESTs. + + The SQL dialect is inferred from the schema_resolver's platform. The + set of supported dialects is the same as sqlglot's. See their + `documentation `_ + for the full list. + + The default_db and default_schema parameters are used to resolve unqualified + table names. For example, the statement "SELECT * FROM my_table" would be + converted to "SELECT * FROM default_db.default_schema.my_table". + + Args: + sql: The SQL statement to parse. This should be a single statement, not + a multi-statement string. + schema_resolver: The schema resolver to use for resolving table schemas. + default_db: The default database to use for unqualified table names. + default_schema: The default schema to use for unqualified table names. + + Returns: + A SqlParsingResult object containing the parsed lineage information. + + The in_tables and out_tables fields contain the input and output tables + for the statement, respectively. These are represented as urns. + The out_tables field will be empty for SELECT statements. + + The column_lineage field contains the column-level lineage information + for the statement. This is a list of ColumnLineageInfo objects, each + representing the lineage for a single output column. The downstream + field contains the output column, and the upstreams field contains the + (urn, column) pairs for the input columns. + + The debug_info field contains debug information about the parsing. If + table_error or column_error are set, then the parsing failed and the + other fields may be incomplete. + """ try: return _sqlglot_lineage_inner( sql=sql, From 8a23c37e7cd628c647a39fcb85b5da90eefdc36b Mon Sep 17 00:00:00 2001 From: John Joyce Date: Tue, 25 Jul 2023 17:17:19 -0700 Subject: [PATCH 183/222] feat(search ui): Adding support for pluggable filter rendering (#8455) --- .../src/app/search/SimpleSearchFilters.tsx | 31 ++++--- .../src/app/search/filters/BasicFilters.tsx | 28 +++++-- .../app/search/filters/MoreFilterOption.tsx | 22 +---- .../src/app/search/filters/MoreFilters.tsx | 34 +++++--- .../app/search/filters/SearchFilterView.tsx | 26 +----- .../search/filters/render/FilterRenderer.tsx | 29 +++++++ .../filters/render/FilterRendererRegistry.tsx | 42 ++++++++++ .../__tests__/FilterRendererRegistry.test.tsx | 80 +++++++++++++++++++ .../render/shared/styledComponents.tsx | 19 +++++ .../src/app/search/filters/render/types.ts | 29 +++++++ .../filters/render/useFilterRenderer.tsx | 17 ++++ .../app/search/filters/styledComponents.ts | 35 ++++++++ 12 files changed, 323 insertions(+), 69 deletions(-) create mode 100644 datahub-web-react/src/app/search/filters/render/FilterRenderer.tsx create mode 100644 datahub-web-react/src/app/search/filters/render/FilterRendererRegistry.tsx create mode 100644 datahub-web-react/src/app/search/filters/render/__tests__/FilterRendererRegistry.test.tsx create mode 100644 datahub-web-react/src/app/search/filters/render/shared/styledComponents.tsx create mode 100644 datahub-web-react/src/app/search/filters/render/types.ts create mode 100644 datahub-web-react/src/app/search/filters/render/useFilterRenderer.tsx diff --git a/datahub-web-react/src/app/search/SimpleSearchFilters.tsx b/datahub-web-react/src/app/search/SimpleSearchFilters.tsx index f6639855f4619..416b04403723f 100644 --- a/datahub-web-react/src/app/search/SimpleSearchFilters.tsx +++ b/datahub-web-react/src/app/search/SimpleSearchFilters.tsx @@ -1,6 +1,8 @@ import * as React from 'react'; import { useEffect, useState } from 'react'; import { FacetFilterInput, FacetMetadata } from '../../types.generated'; +import { FilterScenarioType } from './filters/render/types'; +import { useFilterRendererRegistry } from './filters/render/useFilterRenderer'; import { SimpleSearchFilter } from './SimpleSearchFilter'; import { ENTITY_FILTER_NAME, ENTITY_INDEX_FILTER_NAME, LEGACY_ENTITY_FILTER_NAME } from './utils/constants'; @@ -53,17 +55,28 @@ export const SimpleSearchFilters = ({ facets, selectedFilters, onFilterSelect, l return TOP_FILTERS.indexOf(facetA.field) - TOP_FILTERS.indexOf(facetB.field); }); + const filterRendererRegistry = useFilterRendererRegistry(); + return ( <> - {sortedFacets.map((facet) => ( - - ))} + {sortedFacets.map((facet) => { + return filterRendererRegistry.hasRenderer(facet.field) ? ( + filterRendererRegistry.render(facet.field, { + scenario: FilterScenarioType.SEARCH_V1, + filter: facet, + activeFilters: selectedFilters, + onChangeFilters: onFilterSelect, + }) + ) : ( + + ); + })} ); }; diff --git a/datahub-web-react/src/app/search/filters/BasicFilters.tsx b/datahub-web-react/src/app/search/filters/BasicFilters.tsx index d0d65fca04c1d..e8f56e5c2cd5e 100644 --- a/datahub-web-react/src/app/search/filters/BasicFilters.tsx +++ b/datahub-web-react/src/app/search/filters/BasicFilters.tsx @@ -22,6 +22,8 @@ import { SEARCH_RESULTS_ADVANCED_SEARCH_ID, SEARCH_RESULTS_FILTERS_ID, } from '../../onboarding/config/SearchOnboardingConfig'; +import { useFilterRendererRegistry } from './render/useFilterRenderer'; +import { FilterScenarioType } from './render/types'; const NUM_VISIBLE_FILTER_DROPDOWNS = 5; @@ -80,19 +82,29 @@ export default function BasicFilters({ const shouldShowMoreDropdown = filters && filters.length > NUM_VISIBLE_FILTER_DROPDOWNS + 1; const visibleFilters = shouldShowMoreDropdown ? filters?.slice(0, NUM_VISIBLE_FILTER_DROPDOWNS) : filters; const hiddenFilters = shouldShowMoreDropdown ? filters?.slice(NUM_VISIBLE_FILTER_DROPDOWNS) : []; + const filterRendererRegistry = useFilterRendererRegistry(); return ( - {visibleFilters?.map((filter) => ( - - ))} + {visibleFilters?.map((filter) => { + return filterRendererRegistry.hasRenderer(filter.field) ? ( + filterRendererRegistry.render(filter.field, { + scenario: FilterScenarioType.SEARCH_V2_PRIMARY, + filter, + activeFilters, + onChangeFilters, + }) + ) : ( + + ); + })} {hiddenFilters && hiddenFilters.length > 0 && ( ` - padding: 5px 12px; - font-size: 14px; - display: flex; - align-items: center; - justify-content: space-between; - cursor: pointer; - - &:hover { - background-color: ${ANTD_GRAY[3]}; - } - - ${(props) => props.isActive && `color: ${props.theme.styles['primary-color']};`} - ${(props) => props.isOpen && `background-color: ${ANTD_GRAY[3]};`} -`; +import { MoreFilterOptionLabel } from './styledComponents'; const IconNameWrapper = styled.span` display: flex; @@ -72,7 +56,7 @@ export default function MoreFilterOption({ filter, activeFilters, onChangeFilter /> )} > - updateIsMenuOpen(!isMenuOpen)} isActive={!!numActiveFilters} isOpen={isMenuOpen} @@ -83,7 +67,7 @@ export default function MoreFilterOption({ filter, activeFilters, onChangeFilter {capitalizeFirstLetterOnly(filter.displayName)} {numActiveFilters ? `(${numActiveFilters}) ` : ''} - + ); } diff --git a/datahub-web-react/src/app/search/filters/MoreFilters.tsx b/datahub-web-react/src/app/search/filters/MoreFilters.tsx index dfadb293f7e5a..12ac7e7378e35 100644 --- a/datahub-web-react/src/app/search/filters/MoreFilters.tsx +++ b/datahub-web-react/src/app/search/filters/MoreFilters.tsx @@ -5,8 +5,10 @@ import styled from 'styled-components'; import { FacetFilterInput, FacetMetadata } from '../../../types.generated'; import MoreFilterOption from './MoreFilterOption'; import { getNumActiveFiltersForGroupOfFilters } from './utils'; -import { DropdownLabel } from './SearchFilterView'; +import { SearchFilterLabel } from './styledComponents'; import useSearchFilterAnalytics from './useSearchFilterAnalytics'; +import { useFilterRendererRegistry } from './render/useFilterRenderer'; +import { FilterScenarioType } from './render/types'; const StyledPlus = styled(PlusOutlined)` svg { @@ -36,6 +38,7 @@ export default function MoreFilters({ filters, activeFilters, onChangeFilters }: const { trackShowMoreEvent } = useSearchFilterAnalytics(); const [isMenuOpen, setIsMenuOpen] = useState(false); const numActiveFilters = getNumActiveFiltersForGroupOfFilters(activeFilters, filters); + const filterRendererRegistry = useFilterRendererRegistry(); function updateFiltersAndClose(newFilters: FacetFilterInput[]) { onChangeFilters(newFilters); @@ -52,23 +55,32 @@ export default function MoreFilters({ filters, activeFilters, onChangeFilters }: trigger={['click']} dropdownRender={() => ( - {filters.map((filter) => ( - - ))} + {filters.map((filter) => { + return filterRendererRegistry.hasRenderer(filter.field) ? ( + filterRendererRegistry.render(filter.field, { + scenario: FilterScenarioType.SEARCH_V2_SECONDARY, + filter, + activeFilters, + onChangeFilters: updateFiltersAndClose, + }) + ) : ( + + ); + })} )} open={isMenuOpen} onOpenChange={onOpenChange} > - + More Filters {numActiveFilters ? `(${numActiveFilters}) ` : ''} - + ); } diff --git a/datahub-web-react/src/app/search/filters/SearchFilterView.tsx b/datahub-web-react/src/app/search/filters/SearchFilterView.tsx index 18348448bc70e..ee448e6e1acf2 100644 --- a/datahub-web-react/src/app/search/filters/SearchFilterView.tsx +++ b/datahub-web-react/src/app/search/filters/SearchFilterView.tsx @@ -1,29 +1,11 @@ import { CaretDownFilled } from '@ant-design/icons'; -import { Button, Dropdown } from 'antd'; +import { Dropdown } from 'antd'; import React from 'react'; import styled from 'styled-components'; import OptionsDropdownMenu from './OptionsDropdownMenu'; import { capitalizeFirstLetterOnly } from '../../shared/textUtil'; import { DisplayedFilterOption } from './mapFilterOption'; -import { ANTD_GRAY } from '../../entity/shared/constants'; - -export const DropdownLabel = styled(Button)<{ isActive: boolean }>` - font-size: 14px; - font-weight: 700; - margin-right: 12px; - border: 1px solid ${ANTD_GRAY[5]}; - border-radius: 8px; - display: flex; - align-items: center; - box-shadow: none; - ${(props) => - props.isActive && - ` - background-color: ${props.theme.styles['primary-color']}; - border: 1px solid ${props.theme.styles['primary-color']}; - color: white; - `} -`; +import { SearchFilterLabel } from './styledComponents'; export const IconWrapper = styled.div` margin-right: 8px; @@ -76,7 +58,7 @@ export default function SearchFilterView({ /> )} > - updateIsMenuOpen(!isMenuOpen)} isActive={!!numActiveFilters} data-testid={`filter-dropdown-${capitalizeFirstLetterOnly(displayName)}`} @@ -84,7 +66,7 @@ export default function SearchFilterView({ {filterIcon && {filterIcon}} {capitalizeFirstLetterOnly(displayName)} {numActiveFilters ? `(${numActiveFilters}) ` : ''} - + ); } diff --git a/datahub-web-react/src/app/search/filters/render/FilterRenderer.tsx b/datahub-web-react/src/app/search/filters/render/FilterRenderer.tsx new file mode 100644 index 0000000000000..bf1aa347f92c5 --- /dev/null +++ b/datahub-web-react/src/app/search/filters/render/FilterRenderer.tsx @@ -0,0 +1,29 @@ +import { FilterRenderProps } from './types'; + +/** + * Base interface used for custom search filter renderers + * + */ +export interface FilterRenderer { + /** + * The filter field that is rendered by this renderer + */ + field: string; + + /** + * Renders the filter + */ + render: (props: FilterRenderProps) => JSX.Element; + + /** + * Ant-design icon associated with the Entity. For a list of all candidate icons, see + * https://ant.design/components/icon/ + */ + icon: () => JSX.Element; + + /** + * Returns a label for rendering the value of a particular field, e.g. for rendering the selected set of filters. + * Currently only for rendering the selected value set below Search V2 top-bar. + */ + valueLabel: (value: string) => JSX.Element; +} diff --git a/datahub-web-react/src/app/search/filters/render/FilterRendererRegistry.tsx b/datahub-web-react/src/app/search/filters/render/FilterRendererRegistry.tsx new file mode 100644 index 0000000000000..0ceed3af25d62 --- /dev/null +++ b/datahub-web-react/src/app/search/filters/render/FilterRendererRegistry.tsx @@ -0,0 +1,42 @@ +import { FilterRenderer } from './FilterRenderer'; +import { FilterRenderProps } from './types'; + +function validatedGet(key: K, map: Map): V { + if (map.has(key)) { + return map.get(key) as V; + } + throw new Error(`Unrecognized key ${key} provided in map ${JSON.stringify(map)}`); +} + +/** + * Serves as a singleton registry for custom filter renderers. + */ +export default class FilterRendererRegistry { + renderers: Array = new Array(); + + fieldNameToRenderer: Map = new Map(); + + register(renderer: FilterRenderer) { + this.renderers.push(renderer); + this.fieldNameToRenderer.set(renderer.field, renderer); + } + + hasRenderer(field: string): boolean { + return this.fieldNameToRenderer.has(field); + } + + render(field: string, props: FilterRenderProps): React.ReactNode { + const renderer = validatedGet(field, this.fieldNameToRenderer); + return renderer.render(props); + } + + getValueLabel(field: string, value: string): React.ReactNode { + const renderer = validatedGet(field, this.fieldNameToRenderer); + return renderer.valueLabel(value); + } + + getIcon(field: string): React.ReactNode { + const renderer = validatedGet(field, this.fieldNameToRenderer); + return renderer.icon(); + } +} diff --git a/datahub-web-react/src/app/search/filters/render/__tests__/FilterRendererRegistry.test.tsx b/datahub-web-react/src/app/search/filters/render/__tests__/FilterRendererRegistry.test.tsx new file mode 100644 index 0000000000000..35d41b3bb71a3 --- /dev/null +++ b/datahub-web-react/src/app/search/filters/render/__tests__/FilterRendererRegistry.test.tsx @@ -0,0 +1,80 @@ +import React from 'react'; +import FilterRendererRegistry from '../FilterRendererRegistry'; + +describe('FilterRendererRegistry', () => { + let registry; + let renderer; + let props; + + beforeEach(() => { + registry = new FilterRendererRegistry(); + + // Mock a FilterRenderer instance + renderer = { + field: 'mockField', + render: jest.fn().mockReturnValue(
Rendered
), + valueLabel: jest.fn().mockReturnValue(
ValueLabel
), + icon: jest.fn().mockReturnValue(
Icon
), + }; + + // Mock FilterRenderProps + props = { + /* assuming some props here */ + }; + }); + + describe('register', () => { + it('should register a new FilterRenderer', () => { + registry.register(renderer); + expect(registry.renderers).toContain(renderer); + expect(registry.fieldNameToRenderer.get(renderer.field)).toBe(renderer); + }); + }); + + describe('hasRenderer', () => { + it('should return true if the renderer for a field is registered', () => { + registry.register(renderer); + expect(registry.hasRenderer(renderer.field)).toBe(true); + }); + + it('should return false if the renderer for a field is not registered', () => { + expect(registry.hasRenderer('nonexistentField')).toBe(false); + }); + }); + + describe('render', () => { + it('should return the result of the renderer render method', () => { + registry.register(renderer); + expect(registry.render(renderer.field, props)).toEqual(
Rendered
); + expect(renderer.render).toHaveBeenCalledWith(props); + }); + + it('should throw an error if the renderer for a field is not registered', () => { + expect(() => registry.render('nonexistentField', props)).toThrow(); + }); + }); + + describe('getValueLabel', () => { + it('should return the result of the renderer valueLabel method', () => { + registry.register(renderer); + expect(registry.getValueLabel(renderer.field, 'mockValue')).toEqual(
ValueLabel
); + expect(renderer.valueLabel).toHaveBeenCalledWith('mockValue'); + }); + + it('should throw an error if the renderer for a field is not registered', () => { + expect(() => registry.getValueLabel('nonexistentField', 'mockValue')).toThrow(); + }); + }); + + describe('getIcon', () => { + it('should return the result of the renderer icon method', () => { + registry.register(renderer); + expect(registry.getIcon(renderer.field)).toEqual(
Icon
); + expect(renderer.icon).toHaveBeenCalled(); + }); + + it('should throw an error if the renderer for a field is not registered', () => { + expect(() => registry.getIcon('nonexistentField')).toThrow(); + }); + }); +}); diff --git a/datahub-web-react/src/app/search/filters/render/shared/styledComponents.tsx b/datahub-web-react/src/app/search/filters/render/shared/styledComponents.tsx new file mode 100644 index 0000000000000..7bcbcca048ac6 --- /dev/null +++ b/datahub-web-react/src/app/search/filters/render/shared/styledComponents.tsx @@ -0,0 +1,19 @@ +import styled from 'styled-components'; +import { MoreFilterOptionLabel } from '../../styledComponents'; + +export const SearchFilterWrapper = styled.div` + padding: 0 25px 15px 25px; +`; + +export const Title = styled.div` + align-items: center; + font-weight: bold; + margin-bottom: 10px; + display: flex; + justify-content: left; + cursor: pointer; +`; + +export const StyledMoreFilterOptionLabel = styled(MoreFilterOptionLabel)` + justify-content: left; +`; diff --git a/datahub-web-react/src/app/search/filters/render/types.ts b/datahub-web-react/src/app/search/filters/render/types.ts new file mode 100644 index 0000000000000..1f61919309260 --- /dev/null +++ b/datahub-web-react/src/app/search/filters/render/types.ts @@ -0,0 +1,29 @@ +import { FacetFilter, FacetFilterInput, FacetMetadata } from '../../../../types.generated'; + +/** + * The scenario in which filter rendering is required. + */ +export enum FilterScenarioType { + /** + * The V1 search experience, including embedded list search. + */ + SEARCH_V1, + /** + * The V2 search + browse experience, NOT inside the "More" dropdown. + */ + SEARCH_V2_PRIMARY, + /** + * The V2 search + browse experience, inside the "More" dropdown. + */ + SEARCH_V2_SECONDARY, +} + +/** + * Props passed to every filter renderer + */ +export interface FilterRenderProps { + scenario: FilterScenarioType; + filter: FacetMetadata; + activeFilters: FacetFilterInput[]; + onChangeFilters: (newFilters: FacetFilter[]) => void; +} diff --git a/datahub-web-react/src/app/search/filters/render/useFilterRenderer.tsx b/datahub-web-react/src/app/search/filters/render/useFilterRenderer.tsx new file mode 100644 index 0000000000000..df94cfe49b546 --- /dev/null +++ b/datahub-web-react/src/app/search/filters/render/useFilterRenderer.tsx @@ -0,0 +1,17 @@ +import FilterRendererRegistry from './FilterRendererRegistry'; + +/** + * Configure the render registry. + */ +const RENDERERS = [ + /* Renderers will be registered here */ +]; +const REGISTRY = new FilterRendererRegistry(); +RENDERERS.forEach((renderer) => REGISTRY.register(renderer)); + +/** + * Used to render custom filter views for a particular facet field. + */ +export const useFilterRendererRegistry = () => { + return REGISTRY; +}; diff --git a/datahub-web-react/src/app/search/filters/styledComponents.ts b/datahub-web-react/src/app/search/filters/styledComponents.ts index dafdb9ede9de0..eb70e0719169e 100644 --- a/datahub-web-react/src/app/search/filters/styledComponents.ts +++ b/datahub-web-react/src/app/search/filters/styledComponents.ts @@ -1,5 +1,40 @@ import { Button } from 'antd'; import styled from 'styled-components'; +import { ANTD_GRAY } from '../../entity/shared/constants'; + +export const SearchFilterLabel = styled(Button)<{ isActive: boolean }>` + font-size: 14px; + font-weight: 700; + margin-right: 12px; + border: 1px solid ${ANTD_GRAY[5]}; + border-radius: 8px; + display: flex; + align-items: center; + box-shadow: none; + ${(props) => + props.isActive && + ` + background-color: ${props.theme.styles['primary-color']}; + border: 1px solid ${props.theme.styles['primary-color']}; + color: white; + `} +`; + +export const MoreFilterOptionLabel = styled.div<{ isActive: boolean; isOpen: boolean }>` + padding: 5px 12px; + font-size: 14px; + display: flex; + align-items: center; + justify-content: space-between; + cursor: pointer; + + &:hover { + background-color: ${ANTD_GRAY[3]}; + } + + ${(props) => props.isActive && `color: ${props.theme.styles['primary-color']};`} + ${(props) => props.isOpen && `background-color: ${ANTD_GRAY[3]};`} +`; export const TextButton = styled(Button)<{ marginTop?: number; height?: number }>` color: ${(props) => props.theme.styles['primary-color']}; From c585a1bcc740099fd0a4b76b91f9455e7c7a284e Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 25 Jul 2023 21:20:27 -0700 Subject: [PATCH 184/222] fix(ingest): hint at --update-golden-files option when tests fail (#8507) --- .../src/datahub/testing/compare_metadata_json.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/metadata-ingestion/src/datahub/testing/compare_metadata_json.py b/metadata-ingestion/src/datahub/testing/compare_metadata_json.py index 466e652277fd1..b017afc8c1448 100644 --- a/metadata-ingestion/src/datahub/testing/compare_metadata_json.py +++ b/metadata-ingestion/src/datahub/testing/compare_metadata_json.py @@ -45,7 +45,7 @@ def assert_metadata_files_equal( if copy_output: shutil.copyfile(str(output_path), str(golden_path) + ".output") - print(f"Copied output file to {golden_path}.output") + logger.info(f"Copied output file to {golden_path}.output") if not update_golden and not golden_exists: raise FileNotFoundError( @@ -77,11 +77,15 @@ def assert_metadata_files_equal( if diff: # Call pytest.fail rather than raise an exception to omit stack trace + message = ( + "Metadata files differ (use `pytest --update-golden-files` to update):\n" + ) if isinstance(diff, MCPDiff): - print(diff.pretty(verbose=True)) - pytest.fail(diff.pretty(), pytrace=False) + logger.error(message + diff.pretty(verbose=True)) + pytest.fail(message + diff.pretty(), pytrace=False) else: - pytest.fail(pprint.pformat(diff), pytrace=False) + logger.error(message + pprint.pformat(diff)) + pytest.fail(message + pprint.pformat(diff), pytrace=False) def diff_metadata_json( From 2495b50f8c96d01a98eff07198dc935f1df5ffc0 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 25 Jul 2023 22:42:22 -0700 Subject: [PATCH 185/222] ci: fix commandLine usage in build.gradle (#8510) --- .../airflow-plugin/build.gradle | 5 +++-- metadata-ingestion/build.gradle | 14 +++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/metadata-ingestion-modules/airflow-plugin/build.gradle b/metadata-ingestion-modules/airflow-plugin/build.gradle index d895e29229c41..336be8fc94d44 100644 --- a/metadata-ingestion-modules/airflow-plugin/build.gradle +++ b/metadata-ingestion-modules/airflow-plugin/build.gradle @@ -25,8 +25,9 @@ task installPackage(type: Exec, dependsOn: environmentSetup) { // Workaround for https://github.com/yaml/pyyaml/issues/601. // See https://github.com/yaml/pyyaml/issues/601#issuecomment-1638509577. // and https://github.com/datahub-project/datahub/pull/8435. - commandLine 'bash', '-x', '-c', "${pip_install_command} install 'Cython<3.0' 'PyYAML<6' --no-build-isolation" - commandLine 'bash', '-x', '-c', "${pip_install_command} -e ." + commandLine 'bash', '-x', '-c', + "${pip_install_command} install 'Cython<3.0' 'PyYAML<6' --no-build-isolation && " + + "${pip_install_command} -e ." } task install(dependsOn: [installPackage]) diff --git a/metadata-ingestion/build.gradle b/metadata-ingestion/build.gradle index a1769df968fef..f636cf25c67f7 100644 --- a/metadata-ingestion/build.gradle +++ b/metadata-ingestion/build.gradle @@ -23,14 +23,17 @@ task checkPythonVersion(type: Exec) { task environmentSetup(type: Exec, dependsOn: checkPythonVersion) { inputs.file file('setup.py') outputs.dir("${venv_name}") - commandLine 'bash', '-c', "${python_executable} -m venv ${venv_name} && ${venv_name}/bin/python -m pip install --upgrade pip wheel 'setuptools>=63.0.0'" + commandLine 'bash', '-c', + "${python_executable} -m venv ${venv_name} && " + + "${venv_name}/bin/python -m pip install --upgrade pip wheel 'setuptools>=63.0.0'" } task runPreFlightScript(type: Exec, dependsOn: environmentSetup) { def sentinel_file = ".preflight_sentinel" outputs.file(sentinel_file) - commandLine "scripts/datahub_preflight.sh" - commandLine 'bash', '-c', "touch ${sentinel_file}" + commandLine 'bash', '-c', + "scripts/datahub_preflight.sh && " + + "touch ${sentinel_file}" } task installPackageOnly(type: Exec, dependsOn: runPreFlightScript) { @@ -38,8 +41,9 @@ task installPackageOnly(type: Exec, dependsOn: runPreFlightScript) { inputs.file file('setup.py') outputs.dir("${venv_name}") outputs.file(sentinel_file) - commandLine 'bash', '-x', '-c', "${venv_name}/bin/pip install -e ." - commandLine 'bash', '-c', "touch ${sentinel_file}" + commandLine 'bash', '-x', '-c', + "${venv_name}/bin/pip install -e . &&" + + "touch ${sentinel_file}" } task installPackage(type: Exec, dependsOn: installPackageOnly) { From 4f961dab2afbd52218da5e1512ba2497b11a8185 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Wed, 26 Jul 2023 10:56:07 -0400 Subject: [PATCH 186/222] fix(ui) Fix broken dataPlatformInstance references in browseV2 (#8485) --- datahub-web-react/src/App.tsx | 2 + .../DataPlatformInstanceEntity.tsx | 61 +++++++++++++++++++ .../src/app/search/sidebar/BrowseNode.tsx | 3 +- .../src/graphql/fragments.graphql | 3 + 4 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 datahub-web-react/src/app/entity/dataPlatformInstance/DataPlatformInstanceEntity.tsx diff --git a/datahub-web-react/src/App.tsx b/datahub-web-react/src/App.tsx index 7b580d3c14643..1d9c9cbd27dbb 100644 --- a/datahub-web-react/src/App.tsx +++ b/datahub-web-react/src/App.tsx @@ -34,6 +34,7 @@ import { ContainerEntity } from './app/entity/container/ContainerEntity'; import GlossaryNodeEntity from './app/entity/glossaryNode/GlossaryNodeEntity'; import { DataPlatformEntity } from './app/entity/dataPlatform/DataPlatformEntity'; import { DataProductEntity } from './app/entity/dataProduct/DataProductEntity'; +import { DataPlatformInstanceEntity } from './app/entity/dataPlatformInstance/DataPlatformInstanceEntity'; /* Construct Apollo Client @@ -116,6 +117,7 @@ const App: React.VFC = () => { register.register(new GlossaryNodeEntity()); register.register(new DataPlatformEntity()); register.register(new DataProductEntity()); + register.register(new DataPlatformInstanceEntity()); return register; }, []); diff --git a/datahub-web-react/src/app/entity/dataPlatformInstance/DataPlatformInstanceEntity.tsx b/datahub-web-react/src/app/entity/dataPlatformInstance/DataPlatformInstanceEntity.tsx new file mode 100644 index 0000000000000..a542e1b52f510 --- /dev/null +++ b/datahub-web-react/src/app/entity/dataPlatformInstance/DataPlatformInstanceEntity.tsx @@ -0,0 +1,61 @@ +import * as React from 'react'; +import { DataPlatformInstance, EntityType } from '../../../types.generated'; +import { Entity } from '../Entity'; +import { GenericEntityProperties } from '../shared/types'; +import { getDataForEntityType } from '../shared/containers/profile/utils'; + +/** + * Definition of the DataHub DataPlatformInstance entity. + * Most of this still needs to be filled out. + */ +export class DataPlatformInstanceEntity implements Entity { + type: EntityType = EntityType.DataPlatformInstance; + + icon = () => { + return <>; + }; + + isSearchEnabled = () => false; + + isBrowseEnabled = () => false; + + isLineageEnabled = () => false; + + getAutoCompleteFieldName = () => 'name'; + + getPathName = () => 'dataPlatformInstance'; + + getEntityName = () => 'Data Platform Instance'; + + getCollectionName = () => 'Data Platform Instances'; + + renderProfile = () => <>; + + getOverridePropertiesFromEntity = (): GenericEntityProperties => { + return {}; + }; + + renderPreview = () => { + return <>; + }; + + renderSearch = () => { + return <>; + }; + + displayName = (data: DataPlatformInstance) => { + return data?.instanceId || data.urn; + }; + + getGenericEntityProperties = (data: DataPlatformInstance) => { + return getDataForEntityType({ + data, + entityType: this.type, + getOverrideProperties: this.getOverridePropertiesFromEntity, + }); + }; + + supportedCapabilities = () => { + return new Set([]); + }; +} diff --git a/datahub-web-react/src/app/search/sidebar/BrowseNode.tsx b/datahub-web-react/src/app/search/sidebar/BrowseNode.tsx index ab066027eff52..2125c7a90a3d6 100644 --- a/datahub-web-react/src/app/search/sidebar/BrowseNode.tsx +++ b/datahub-web-react/src/app/search/sidebar/BrowseNode.tsx @@ -21,6 +21,7 @@ import { } from './BrowseContext'; import useSidebarAnalytics from './useSidebarAnalytics'; import EntityLink from './EntityLink'; +import { EntityType } from '../../../types.generated'; const FolderStyled = styled(FolderOutlined)` font-size: 16px; @@ -42,7 +43,7 @@ const BrowseNode = () => { const platformAggregation = usePlatformAggregation(); const browseResultGroup = useBrowseResultGroup(); const { count, entity } = browseResultGroup; - const hasEntityLink = !!entity; + const hasEntityLink = !!entity && entity.type !== EntityType.DataPlatformInstance; const displayName = useBrowseDisplayName(); const { trackSelectNodeEvent, trackToggleNodeEvent } = useSidebarAnalytics(); diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql index f0e6841c9f86c..219722ad1645a 100644 --- a/datahub-web-react/src/graphql/fragments.graphql +++ b/datahub-web-react/src/graphql/fragments.graphql @@ -1112,4 +1112,7 @@ fragment entityDisplayNameFields on Entity { name } } + ... on DataPlatformInstance { + instanceId + } } From 7bbeedddc4bd51791ab952e8649193fc5f02f2ff Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Wed, 26 Jul 2023 10:56:40 -0400 Subject: [PATCH 187/222] fix(dataProduct) Show entity count excluding soft deleted entities (#8444) Co-authored-by: Joshua Eilers --- .../com/linkedin/datahub/graphql/GmsGraphQLEngine.java | 8 ++++++++ .../dataproduct/ListDataProductAssetsResolver.java | 5 ++++- datahub-graphql-core/src/main/resources/entity.graphql | 7 ++++++- .../src/app/entity/dataProduct/DataProductEntity.tsx | 6 +++--- datahub-web-react/src/graphql/dataProduct.graphql | 4 +++- 5 files changed, 24 insertions(+), 6 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index f22568602d6b4..d6dd2de6d31e3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -645,6 +645,7 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) { configureOrganisationRoleResolvers(builder); configureGlossaryNodeResolvers(builder); configureDomainResolvers(builder); + configureDataProductResolvers(builder); configureAssertionResolvers(builder); configurePolicyResolvers(builder); configureDataProcessInstanceResolvers(builder); @@ -1679,6 +1680,13 @@ private void configureDomainResolvers(final RuntimeWiring.Builder builder) { ); } + private void configureDataProductResolvers(final RuntimeWiring.Builder builder) { + builder.type("DataProduct", typeWiring -> typeWiring + .dataFetcher("entities", new ListDataProductAssetsResolver(this.entityClient)) + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + ); + } + private void configureAssertionResolvers(final RuntimeWiring.Builder builder) { builder.type("Assertion", typeWiring -> typeWiring.dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java index c1c37868401e7..f7db6e56940d7 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java @@ -5,6 +5,7 @@ import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.DataMap; import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.DataProduct; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.SearchAcrossEntitiesInput; import com.linkedin.datahub.graphql.generated.SearchResults; @@ -50,7 +51,9 @@ public class ListDataProductAssetsResolver implements DataFetcher get(DataFetchingEnvironment environment) { final QueryContext context = environment.getContext(); - final Urn dataProductUrn = UrnUtils.getUrn(environment.getArgument("urn")); + // get urn from either input or source (in the case of "entities" field) + final String urn = environment.getArgument("urn") != null ? environment.getArgument("urn") : ((DataProduct) environment.getSource()).getUrn(); + final Urn dataProductUrn = UrnUtils.getUrn(urn); final SearchAcrossEntitiesInput input = bindArgument(environment.getArgument("input"), SearchAcrossEntitiesInput.class); diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 59e35d5bc3ff4..d4745af33e0ce 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -10890,6 +10890,11 @@ type DataProduct implements Entity { """ relationships(input: RelationshipsInput!): EntityRelationshipsResult + """ + Children entities inside of the DataProduct + """ + entities(input: SearchAcrossEntitiesInput): SearchResults + """ The structured glossary terms associated with the Data Product """ @@ -10926,7 +10931,7 @@ type DataProductProperties { externalUrl: String """ - Number of children entities inside of the Data Product + Number of children entities inside of the Data Product. This number includes soft deleted entities. """ numAssets: Int diff --git a/datahub-web-react/src/app/entity/dataProduct/DataProductEntity.tsx b/datahub-web-react/src/app/entity/dataProduct/DataProductEntity.tsx index a20aa017f21a7..faa254cce73a6 100644 --- a/datahub-web-react/src/app/entity/dataProduct/DataProductEntity.tsx +++ b/datahub-web-react/src/app/entity/dataProduct/DataProductEntity.tsx @@ -132,7 +132,7 @@ export class DataProductEntity implements Entity { globalTags={data.tags} glossaryTerms={data.glossaryTerms} domain={data.domain?.domain} - entityCount={data?.properties?.numAssets || undefined} + entityCount={data?.entities?.total || undefined} externalUrl={data.properties?.externalUrl} /> ); @@ -149,7 +149,7 @@ export class DataProductEntity implements Entity { globalTags={data.tags} glossaryTerms={data.glossaryTerms} domain={data.domain?.domain} - entityCount={data?.properties?.numAssets || undefined} + entityCount={data?.entities?.total || undefined} externalUrl={data.properties?.externalUrl} /> ); @@ -162,7 +162,7 @@ export class DataProductEntity implements Entity { getOverridePropertiesFromEntity = (data: DataProduct) => { const name = data?.properties?.name; const externalUrl = data?.properties?.externalUrl; - const entityCount = data?.properties?.numAssets || undefined; + const entityCount = data?.entities?.total || undefined; return { name, externalUrl, diff --git a/datahub-web-react/src/graphql/dataProduct.graphql b/datahub-web-react/src/graphql/dataProduct.graphql index 994f43a0e8a0f..464ab7cc12164 100644 --- a/datahub-web-react/src/graphql/dataProduct.graphql +++ b/datahub-web-react/src/graphql/dataProduct.graphql @@ -24,7 +24,6 @@ fragment dataProductSearchFields on DataProduct { name description externalUrl - numAssets } ownership { ...ownershipFields @@ -38,6 +37,9 @@ fragment dataProductSearchFields on DataProduct { domain { ...entityDomain } + entities(input: { start: 0, count: 0, query: "*" }) { + total + } } mutation createDataProduct($input: CreateDataProductInput!) { From 2bd8be47907b614cd259a031f7fd8b5f1a073005 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Wed, 26 Jul 2023 09:13:45 -0700 Subject: [PATCH 188/222] feat(ui): Adding support for rendering assertion health status in Dataset Search Card, Search Preview, etc. (#8460) --- .../dataset/DatasetHealthResolver.java | 4 +- .../src/app/entity/dataset/DatasetEntity.tsx | 2 + .../app/entity/dataset/preview/Preview.tsx | 6 +- .../components/styled/DeprecationPill.tsx | 33 ++++---- .../styled/search/EmbeddedListSearch.tsx | 25 +++++- .../styled/search/EmbeddedListSearchModal.tsx | 8 +- .../search/EmbeddedListSearchResults.tsx | 5 +- .../profile/header/EntityHeader.tsx | 18 ++-- .../profile/header/EntityHealth.tsx | 36 ++++++++ .../profile/header/EntityHealthPopover.tsx | 82 +++++++++++++++++++ .../profile/header/EntityHealthStatus.tsx | 36 ++++++-- .../src/app/ingest/source/IngestedAssets.tsx | 1 + .../src/app/preview/DefaultPreviewCard.tsx | 7 +- .../renderer/component/EntityNameList.tsx | 15 +++- .../src/app/shared/health/healthUtils.tsx | 68 ++++++++++++++- datahub-web-react/src/graphql/search.graphql | 6 ++ 16 files changed, 301 insertions(+), 51 deletions(-) create mode 100644 datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHealth.tsx create mode 100644 datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHealthPopover.tsx diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetHealthResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetHealthResolver.java index 729446eab1fa9..1587df4c9899b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetHealthResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetHealthResolver.java @@ -153,12 +153,12 @@ private Health computeAssertionHealthForDataset(final String datasetUrn, final Q health.setType(HealthStatusType.ASSERTIONS); if (failingAssertionUrns.size() > 0) { health.setStatus(HealthStatus.FAIL); - health.setMessage(String.format("Dataset is failing %s/%s assertions.", failingAssertionUrns.size(), + health.setMessage(String.format("%s of %s assertions are failing", failingAssertionUrns.size(), activeAssertionUrns.size())); health.setCauses(failingAssertionUrns); } else { health.setStatus(HealthStatus.PASS); - health.setMessage("Dataset is passing all assertions."); + health.setMessage("All assertions are passing"); } return health; diff --git a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx index cf9e70643aea7..b55b4c54951ef 100644 --- a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx +++ b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx @@ -257,6 +257,7 @@ export class DatasetEntity implements Entity { dataProduct={getDataProduct(genericProperties?.dataProduct)} container={data.container} externalUrl={data.properties?.externalUrl} + health={data.health} /> ); }; @@ -299,6 +300,7 @@ export class DatasetEntity implements Entity { lastUpdatedMs={ (data as any).lastOperation?.length && (data as any).lastOperation[0].lastUpdatedTimestamp } + health={data.health} /> ); }; diff --git a/datahub-web-react/src/app/entity/dataset/preview/Preview.tsx b/datahub-web-react/src/app/entity/dataset/preview/Preview.tsx index 6a31e5cd19fbc..fd2583e4f5982 100644 --- a/datahub-web-react/src/app/entity/dataset/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/dataset/preview/Preview.tsx @@ -13,6 +13,7 @@ import { Deprecation, DatasetStatsSummary, DataProduct, + Health, } from '../../../../types.generated'; import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; import { useEntityRegistry } from '../../../useEntityRegistry'; @@ -47,6 +48,7 @@ export const Preview = ({ sizeInBytes, statsSummary, lastUpdatedMs, + health, }: { urn: string; name: string; @@ -74,6 +76,7 @@ export const Preview = ({ sizeInBytes?: number | null; statsSummary?: DatasetStatsSummary | null; lastUpdatedMs?: number | null; + health?: Health[] | null; }): JSX.Element => { const entityRegistry = useEntityRegistry(); return ( @@ -107,11 +110,10 @@ export const Preview = ({ rowCount={rowCount} columnCount={columnCount} sizeInBytes={sizeInBytes} - queryCountLast30Days={statsSummary?.queryCountLast30Days} - uniqueUserCountLast30Days={statsSummary?.uniqueUserCountLast30Days} lastUpdatedMs={lastUpdatedMs} /> } + health={health || undefined} /> ); }; diff --git a/datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx b/datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx index 79c64bde34c47..ffc32c1538259 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx +++ b/datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx @@ -10,22 +10,25 @@ import { ANTD_GRAY } from '../../constants'; import { useBatchUpdateDeprecationMutation } from '../../../../../graphql/mutations.generated'; const DeprecatedContainer = styled.div` - width: 104px; height: 18px; - border: 1px solid #ef5b5b; + border: 1px solid #cd0d24; border-radius: 15px; display: flex; justify-content: center; align-items: center; - color: #ef5b5b; + color: #cd0d24; margin-left: 0px; - padding-top: 12px; - padding-bottom: 12px; + margin-right: 8px; + padding-top: 8px; + padding-bottom: 8px; + padding-right: 4px; + padding-left: 4px; `; const DeprecatedText = styled.div` - color: #ef5b5b; - margin-left: 5px; + padding-right: 2px; + padding-left: 2px; + font-size: 10px; `; const DeprecatedTitle = styled(Typography.Text)` @@ -53,10 +56,6 @@ const ThinDivider = styled(Divider)` margin-bottom: 8px; `; -const StyledInfoCircleOutlined = styled(InfoCircleOutlined)` - color: #ef5b5b; -`; - const UndeprecatedIcon = styled(InfoCircleOutlined)` font-size: 14px; padding-right: 6px; @@ -74,12 +73,11 @@ const IconGroup = styled.div` type Props = { urn: string; deprecation: Deprecation; - preview?: boolean | null; refetch?: () => void; showUndeprecate: boolean | null; }; -export const DeprecationPill = ({ deprecation, preview, urn, refetch, showUndeprecate }: Props) => { +export const DeprecationPill = ({ deprecation, urn, refetch, showUndeprecate }: Props) => { const [batchUpdateDeprecationMutation] = useBatchUpdateDeprecationMutation(); /** * Deprecation Decommission Timestamp @@ -166,12 +164,9 @@ export const DeprecationPill = ({ deprecation, preview, urn, refetch, showUndepr ) } > - {(preview && ) || ( - - - Deprecated - - )} + + DEPRECATED + ); }; diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearch.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearch.tsx index dbeac480d5195..da031a532e04c 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearch.tsx +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearch.tsx @@ -1,7 +1,12 @@ import React, { useState, useEffect } from 'react'; import styled from 'styled-components'; import { ApolloError } from '@apollo/client'; -import { FacetFilterInput, FacetMetadata, SearchAcrossEntitiesInput } from '../../../../../../types.generated'; +import { + EntityType, + FacetFilterInput, + FacetMetadata, + SearchAcrossEntitiesInput, +} from '../../../../../../types.generated'; import { UnionType } from '../../../../../search/utils/constants'; import { SearchCfg } from '../../../../../../conf'; import { EmbeddedListSearchResults } from './EmbeddedListSearchResults'; @@ -11,6 +16,7 @@ import { FilterSet, GetSearchResultsParams, SearchResultsInterface } from './typ import { isListSubset } from '../../../utils'; import { EntityAndType } from '../../../types'; import { Message } from '../../../../../shared/Message'; +import { EntityActionProps } from '../../../../../recommendations/renderer/component/EntityNameList'; import { generateOrFilters } from '../../../../../search/utils/generateOrFilters'; import { mergeFilterSets } from '../../../../../search/utils/filterUtils'; import { useDownloadScrollAcrossEntitiesSearchResults } from '../../../../../search/utils/useDownloadScrollAcrossEntitiesSearchResults'; @@ -64,6 +70,7 @@ export const removeFixedFiltersFromFacets = (fixedFilters: FilterSet, facets: Fa type Props = { query: string; + entityTypes?: EntityType[]; page: number; unionType: UnionType; filters: FacetFilterInput[]; @@ -71,6 +78,7 @@ type Props = { onChangeFilters: (filters) => void; onChangePage: (page) => void; onChangeUnionType: (unionType: UnionType) => void; + onTotalChanged?: (newTotal: number) => void; emptySearchQuery?: string | null; fixedFilters?: FilterSet; fixedQuery?: string | null; @@ -79,6 +87,7 @@ type Props = { defaultFilters?: Array; searchBarStyle?: any; searchBarInputStyle?: any; + entityAction?: React.FC; skipCache?: boolean; useGetSearchResults?: (params: GetSearchResultsParams) => { data: SearchResultsInterface | undefined | null; @@ -98,6 +107,7 @@ type Props = { export const EmbeddedListSearch = ({ query, + entityTypes, filters, page, unionType, @@ -105,6 +115,7 @@ export const EmbeddedListSearch = ({ onChangeFilters, onChangePage, onChangeUnionType, + onTotalChanged, emptySearchQuery, fixedFilters, fixedQuery, @@ -113,6 +124,7 @@ export const EmbeddedListSearch = ({ defaultFilters, searchBarStyle, searchBarInputStyle, + entityAction, skipCache, useGetSearchResults = useWrappedSearchResults, useGetDownloadSearchResults = useDownloadScrollAcrossEntitiesSearchResults, @@ -143,7 +155,7 @@ export const EmbeddedListSearch = ({ const { refetch: refetchForDownload } = useGetDownloadSearchResults({ variables: { input: { - types: [], + types: entityTypes || [], query, count: SearchCfg.RESULTS_PER_PAGE, orFilters: generateOrFilters(unionType, filters), @@ -154,7 +166,7 @@ export const EmbeddedListSearch = ({ }); let searchInput: SearchAcrossEntitiesInput = { - types: [], + types: entityTypes || [], query: finalQuery, start: (page - 1) * numResultsPerPage, count: numResultsPerPage, @@ -188,6 +200,12 @@ export const EmbeddedListSearch = ({ } }); + useEffect(() => { + if (data?.total !== undefined && onTotalChanged) { + onTotalChanged(data?.total); + } + }, [data?.total, onTotalChanged]); + const searchResultEntities = data?.searchResults?.map((result) => ({ urn: result.entity.urn, type: result.entity.type })) || []; const searchResultUrns = searchResultEntities.map((entity) => entity.urn); @@ -276,6 +294,7 @@ export const EmbeddedListSearch = ({ isSelectMode={isSelectMode} selectedEntities={selectedEntities} setSelectedEntities={setSelectedEntities} + entityAction={entityAction} /> ); diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchModal.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchModal.tsx index b2038ff4d6d03..3a5fdbf4fba4e 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchModal.tsx +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchModal.tsx @@ -3,6 +3,7 @@ import { Button, Modal } from 'antd'; import styled from 'styled-components'; import { FacetFilterInput } from '../../../../../../types.generated'; import { EmbeddedListSearch } from './EmbeddedListSearch'; +import { EntityActionProps } from '../../../../../recommendations/renderer/component/EntityNameList'; import { UnionType } from '../../../../../search/utils/constants'; import { FilterSet } from './types'; @@ -18,6 +19,7 @@ const modalBodyStyle = { }; type Props = { + title: React.ReactNode; emptySearchQuery?: string | null; fixedFilters?: FilterSet; fixedQuery?: string | null; @@ -27,9 +29,11 @@ type Props = { onClose?: () => void; searchBarStyle?: any; searchBarInputStyle?: any; + entityAction?: React.FC; }; export const EmbeddedListSearchModal = ({ + title, emptySearchQuery, fixedFilters, fixedQuery, @@ -39,6 +43,7 @@ export const EmbeddedListSearchModal = ({ onClose, searchBarStyle, searchBarInputStyle, + entityAction, }: Props) => { // Component state const [query, setQuery] = useState(''); @@ -64,7 +69,7 @@ export const EmbeddedListSearchModal = ({ width={800} style={modalStyle} bodyStyle={modalBodyStyle} - title="View Ingested Assets" + title={title} visible onCancel={onClose} footer={} @@ -87,6 +92,7 @@ export const EmbeddedListSearchModal = ({ defaultFilters={defaultFilters} searchBarStyle={searchBarStyle} searchBarInputStyle={searchBarInputStyle} + entityAction={entityAction} /> diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchResults.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchResults.tsx index 99e6410aeb538..893e66fcf7758 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchResults.tsx +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchResults.tsx @@ -3,7 +3,7 @@ import { Pagination, Typography } from 'antd'; import styled from 'styled-components'; import { FacetFilterInput, FacetMetadata, SearchResults as SearchResultType } from '../../../../../../types.generated'; import { SearchCfg } from '../../../../../../conf'; -import { EntityNameList } from '../../../../../recommendations/renderer/component/EntityNameList'; +import { EntityNameList, EntityActionProps } from '../../../../../recommendations/renderer/component/EntityNameList'; import { ReactComponent as LoadingSvg } from '../../../../../../images/datahub-logo-color-loading_pendulum.svg'; import { EntityAndType } from '../../../types'; import { UnionType } from '../../../../../search/utils/constants'; @@ -74,6 +74,7 @@ interface Props { setSelectedEntities: (entities: EntityAndType[]) => any; numResultsPerPage: number; setNumResultsPerPage: (numResults: number) => void; + entityAction?: React.FC; } export const EmbeddedListSearchResults = ({ @@ -92,6 +93,7 @@ export const EmbeddedListSearchResults = ({ setSelectedEntities, numResultsPerPage, setNumResultsPerPage, + entityAction, }: Props) => { const pageStart = searchResponse?.start || 0; const pageSize = searchResponse?.count || 0; @@ -135,6 +137,7 @@ export const EmbeddedListSearchResults = ({ selectedEntities={selectedEntities} setSelectedEntities={setSelectedEntities} bordered={false} + entityAction={entityAction} /> )} diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx index 9bd3e836f86f7..97595a515b34d 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx @@ -1,21 +1,21 @@ import React from 'react'; import styled from 'styled-components/macro'; import { useEntityData, useRefetch } from '../../../EntityContext'; -import { EntityHealthStatus } from './EntityHealthStatus'; import EntityDropdown, { EntityMenuItems } from '../../../EntityDropdown/EntityDropdown'; import PlatformContent from './PlatformContent'; import { getPlatformName } from '../../../utils'; import { EntityType, PlatformPrivileges } from '../../../../../../types.generated'; import EntityCount from './EntityCount'; +import { EntityHealth } from './EntityHealth'; import EntityName from './EntityName'; import { DeprecationPill } from '../../../components/styled/DeprecationPill'; -import CompactContext from '../../../../../shared/CompactContext'; import { EntitySubHeaderSection, GenericEntityProperties } from '../../../types'; import EntityActions, { EntityActionItem } from '../../../entity/EntityActions'; import ExternalUrlButton from '../../../ExternalUrlButton'; import ShareButton from '../../../../../shared/share/ShareButton'; import { capitalizeFirstLetterOnly } from '../../../../../shared/textUtil'; import { useUserContext } from '../../../../../context/useUserContext'; +import { useEntityRegistry } from '../../../../../useEntityRegistry'; const TitleWrapper = styled.div` display: flex; @@ -87,13 +87,13 @@ export const EntityHeader = ({ headerDropdownItems, headerActionItems, isNameEdi const platformName = getPlatformName(entityData); const externalUrl = entityData?.externalUrl || undefined; const entityCount = entityData?.entityCount; - const isCompact = React.useContext(CompactContext); const entityName = entityData?.name; const subType = capitalizeFirstLetterOnly(entityData?.subTypes?.typeNames?.[0]) || undefined; const canEditName = isNameEditable && getCanEditName(entityType, entityData, me?.platformPrivileges as PlatformPrivileges); + const entityRegistry = useEntityRegistry(); return ( <> @@ -107,17 +107,15 @@ export const EntityHeader = ({ headerDropdownItems, headerActionItems, isNameEdi urn={urn} deprecation={entityData?.deprecation} showUndeprecate - preview={isCompact} refetch={refetch} /> )} - {entityData?.health?.map((health) => ( - - ))} + )} diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHealth.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHealth.tsx new file mode 100644 index 0000000000000..baef67a3d1c88 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHealth.tsx @@ -0,0 +1,36 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Link } from 'react-router-dom'; +import { Health } from '../../../../../../types.generated'; +import { getHealthSummaryIcon, isUnhealthy } from '../../../../../shared/health/healthUtils'; +import { EntityHealthPopover } from './EntityHealthPopover'; + +const Container = styled.div` + display: flex; + justify-content: center; + align-items: center; +`; + +type Props = { + health: Health[]; + baseUrl: string; +}; + +export const EntityHealth = ({ health, baseUrl }: Props) => { + const unhealthy = isUnhealthy(health); + const icon = getHealthSummaryIcon(health); + return ( + <> + {(unhealthy && ( + + + + {icon} + + + + )) || + undefined} + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHealthPopover.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHealthPopover.tsx new file mode 100644 index 0000000000000..fac81c2066141 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHealthPopover.tsx @@ -0,0 +1,82 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Popover, Divider } from 'antd'; +import { + getHealthSummaryIcon, + getHealthSummaryMessage, + HealthSummaryIconType, +} from '../../../../../shared/health/healthUtils'; +import { EntityHealthStatus } from './EntityHealthStatus'; +import { Health } from '../../../../../../types.generated'; +import { ANTD_GRAY } from '../../../constants'; + +const Header = styled.span` + display: flex; + align-items: center; + justify-content: left; +`; + +const Icon = styled.span` + margin-right: 8px; + display: flex; + align-items: center; +`; + +const Title = styled.span` + font-weight: bold; + color: ${ANTD_GRAY[1]}; + padding-top: 4px; + padding-bottom: 4px; + font-size: 14px; +`; + +const StatusContainer = styled.div` + margin-bottom: 8px; +`; + +const StyledDivider = styled(Divider)` + &&& { + margin: 0px; + padding: 0px; + padding-right: 8px; + padding-left: 8px; + margin-top: 8px; + margin-bottom: 8px; + border-color: ${ANTD_GRAY[5]}; + } +`; + +const popoverStyle = { backgroundColor: '#262626' }; + +type Props = { + health: Health[]; + baseUrl: string; + children: React.ReactNode; +}; + +export const EntityHealthPopover = ({ health, baseUrl, children }: Props) => { + const icon = getHealthSummaryIcon(health, HealthSummaryIconType.OUTLINED); + const message = getHealthSummaryMessage(health); + return ( + +
+ {icon} {message} +
+ + {health.map((h) => ( + + + + ))} + + } + overlayStyle={popoverStyle} + placement="right" + zIndex={10000000} + > + {children} + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHealthStatus.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHealthStatus.tsx index 4d16fc9ac0bff..601906038d3b7 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHealthStatus.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHealthStatus.tsx @@ -1,27 +1,45 @@ import React from 'react'; -import { Tooltip } from 'antd'; import styled from 'styled-components'; -import { getHealthIcon } from '../../../../../shared/health/healthUtils'; -import { HealthStatus, HealthStatusType } from '../../../../../../types.generated'; +import { Link } from 'react-router-dom'; +import { getHealthTypeName, getHealthRedirectPath } from '../../../../../shared/health/healthUtils'; +import { HealthStatusType } from '../../../../../../types.generated'; +import { ANTD_GRAY, REDESIGN_COLORS } from '../../../constants'; const StatusContainer = styled.div` display: flex; - justify-content: center; + justify-content: left; align-items: center; - margin-left: 8px; + color: ${ANTD_GRAY[1]}; + font-size: 14px; +`; + +const Title = styled.span` + display: flex; + align-items: center; + font-weight: bold; + margin-right: 8px; + width: 72px; +`; + +const RedirectLink = styled(Link)` + margin-left: 4px; + color: ${REDESIGN_COLORS.BLUE}; `; type Props = { type: HealthStatusType; - status: HealthStatus; message?: string | undefined; + baseUrl: string; }; -export const EntityHealthStatus = ({ type, status, message }: Props) => { - const icon = getHealthIcon(type, status, 18); +export const EntityHealthStatus = ({ type, message, baseUrl }: Props) => { + const title = getHealthTypeName(type); + const redirectPath = getHealthRedirectPath(type); + const fullPath = `${baseUrl}/${redirectPath}`; return ( - {icon} + {title} {message} + {redirectPath && details} ); }; diff --git a/datahub-web-react/src/app/ingest/source/IngestedAssets.tsx b/datahub-web-react/src/app/ingest/source/IngestedAssets.tsx index 790339dcf5723..5c138b809498b 100644 --- a/datahub-web-react/src/app/ingest/source/IngestedAssets.tsx +++ b/datahub-web-react/src/app/ingest/source/IngestedAssets.tsx @@ -136,6 +136,7 @@ export default function IngestedAssets({ id }: Props) { {showAssetSearch && ( ; paths?: EntityPath[]; + health?: Health[]; } export default function DefaultPreviewCard({ @@ -229,6 +232,7 @@ export default function DefaultPreviewCard({ logoUrls, previewType, paths, + health, }: Props) { // sometimes these lists will be rendered inside an entity container (for example, in the case of impact analysis) // in those cases, we may want to enrich the preview w/ context about the container entity @@ -288,8 +292,9 @@ export default function DefaultPreviewCard({ )} {deprecation?.deprecated && ( - + )} + {health && health.length && } {externalUrl && ( any; bordered?: boolean; + entityAction?: React.FC; }; export const EntityNameList = ({ @@ -85,6 +91,7 @@ export const EntityNameList = ({ selectedEntities = [], setSelectedEntities, bordered = true, + entityAction, }: Props) => { const entityRegistry = useEntityRegistry(); const selectedEntityUrns = selectedEntities?.map((entity) => entity.urn) || []; @@ -111,6 +118,8 @@ export const EntityNameList = ({ } }; + const EntityAction = entityAction as React.FC; + return ( @@ -155,7 +166,9 @@ export const EntityNameList = ({ degree={additionalProperties?.degree} deprecation={deprecation} paths={additionalProperties?.paths} + health={health || undefined} /> + {entityAction && } diff --git a/datahub-web-react/src/app/shared/health/healthUtils.tsx b/datahub-web-react/src/app/shared/health/healthUtils.tsx index e43abee54fd72..823d77d7eabe9 100644 --- a/datahub-web-react/src/app/shared/health/healthUtils.tsx +++ b/datahub-web-react/src/app/shared/health/healthUtils.tsx @@ -1,6 +1,50 @@ -import { CheckOutlined, CloseOutlined, WarningOutlined } from '@ant-design/icons'; +import { + CheckOutlined, + CloseOutlined, + ExclamationCircleOutlined, + ExclamationCircleTwoTone, + WarningOutlined, +} from '@ant-design/icons'; import React from 'react'; -import { HealthStatus, HealthStatusType } from '../../../types.generated'; +import styled from 'styled-components'; +import { HealthStatus, HealthStatusType, Health } from '../../../types.generated'; + +const HEALTH_INDICATOR_COLOR = '#d48806'; + +const UnhealthyIconFilled = styled(ExclamationCircleTwoTone)` + font-size: 16px; +`; + +const UnhealthyIconOutlined = styled(ExclamationCircleOutlined)` + color: ${HEALTH_INDICATOR_COLOR}; + font-size: 16px; +`; + +export enum HealthSummaryIconType { + OUTLINED, + FILLED, +} + +export const isUnhealthy = (healths: Health[]) => { + const assertionHealth = healths.find((health) => health.type === HealthStatusType.Assertions); + const isFailingAssertions = assertionHealth?.status === HealthStatus.Fail; + // Currently, being unhealthy is defined as having failing assertions. + return isFailingAssertions; +}; + +export const getHealthSummaryIcon = (healths: Health[], type: HealthSummaryIconType = HealthSummaryIconType.FILLED) => { + const unhealthy = isUnhealthy(healths); + return unhealthy + ? (type === HealthSummaryIconType.FILLED && ) || ( + + ) + : undefined; +}; + +export const getHealthSummaryMessage = (healths: Health[]) => { + const unhealthy = isUnhealthy(healths); + return unhealthy ? 'This asset may be unhealthy' : 'This asset is healthy'; +}; export const getHealthColor = (status: HealthStatus) => { switch (status) { @@ -43,3 +87,23 @@ export const getHealthIcon = (type: HealthStatusType, status: HealthStatus, font throw new Error(`Unrecognized Health Status Type ${type} provided`); } }; + +export const getHealthRedirectPath = (type: HealthStatusType) => { + switch (type) { + case HealthStatusType.Assertions: { + return 'Validation/Assertions'; + } + default: + throw new Error(`Unrecognized Health Status Type ${type} provided`); + } +}; + +export const getHealthTypeName = (type: HealthStatusType) => { + switch (type) { + case HealthStatusType.Assertions: { + return 'Assertions'; + } + default: + throw new Error(`Unrecognized Health Status Type ${type} provided`); + } +}; diff --git a/datahub-web-react/src/graphql/search.graphql b/datahub-web-react/src/graphql/search.graphql index 7bc9f538ab206..6b922af1b7db9 100644 --- a/datahub-web-react/src/graphql/search.graphql +++ b/datahub-web-react/src/graphql/search.graphql @@ -299,6 +299,12 @@ fragment searchResultFields on Entity { deprecation { ...deprecationFields } + health { + type + status + message + causes + } siblings { isPrimary siblings { From 5a3f91de536873b0c3a07e03da04c5619d5350b1 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Thu, 27 Jul 2023 17:01:39 +0530 Subject: [PATCH 189/222] =?UTF-8?q?docs(ingest/bigquery):=20add=20permissi?= =?UTF-8?q?ons=20to=20profile=20google=20drive=20backed=E2=80=A6=20(#8490)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metadata-ingestion/docs/sources/bigquery/bigquery_pre.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metadata-ingestion/docs/sources/bigquery/bigquery_pre.md b/metadata-ingestion/docs/sources/bigquery/bigquery_pre.md index 2217e4185d4b7..0d856b915629d 100644 --- a/metadata-ingestion/docs/sources/bigquery/bigquery_pre.md +++ b/metadata-ingestion/docs/sources/bigquery/bigquery_pre.md @@ -85,6 +85,10 @@ If you have multiple projects in your BigQuery setup, the role should be granted client_id: "123456678890" ``` +##### Profiling Requirements + +To profile BigQuery external tables backed by Google Drive document, you need to grant document's "Viewer" access to service account's email address (`client_email` in credentials json file). To find the Google Drive document linked to BigQuery table, open the BigQuery console, locate the needed table, select "Details" from the drop-down menu in the top-right corner and refer "Source" field . To share access of Google Drive document, open the document, click "Share" in the top-right corner, add the service account's email address that needs "Viewer" access. ![Google Drive Sharing Dialog](https://github.com/datahub-project/static-assets/raw/main/imgs/integrations/bigquery/google_drive_share.png) + ### Lineage Computation Details When `use_exported_bigquery_audit_metadata` is set to `true`, lineage information will be computed using exported bigquery logs. On how to setup exported bigquery audit logs, refer to the following [docs](https://cloud.google.com/bigquery/docs/reference/auditlogs#defining_a_bigquery_log_sink_using_gcloud) on BigQuery audit logs. Note that only protoPayloads with "type.googleapis.com/google.cloud.audit.BigQueryAuditMetadata" are supported by the current ingestion version. The `bigquery_audit_metadata_datasets` parameter will be used only if `use_exported_bigquery_audit_metadat` is set to `true`. From b9060db5153ecd140b34aa7485519f1e04f7a9e3 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Thu, 27 Jul 2023 17:02:48 +0530 Subject: [PATCH 190/222] chore(ingest/tableau): miscellaneous cleanup refractor (#8417) Co-authored-by: Andrew Sikowitz --- .../src/datahub/ingestion/source/tableau.py | 178 +- .../ingestion/source/tableau_constant.py | 1 - ...bleau_extract_all_project_mces_golden.json | 773 +- .../tableau/tableau_mces_golden.json | 773 +- .../tableau_mces_golden_deleted_stateful.json | 260 +- .../tableau_nested_project_mces_golden.json | 773 +- .../tableau_signout_timeout_mces_golden.json | 773 +- .../tableau/tableau_state_mces_golden.json | 32464 ---------------- ...au_with_platform_instance_mces_golden.json | 773 +- 9 files changed, 399 insertions(+), 36369 deletions(-) delete mode 100644 metadata-ingestion/tests/integration/tableau/tableau_state_mces_golden.json diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau.py index 67bd1af6c2d7f..e923727a990f4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau.py @@ -613,7 +613,6 @@ def _init_datasource_registry(self) -> None: ) continue self.datasource_project_map[ds.id] = ds.project_id - self.report.get_all_datasources_query_failed = False except Exception as e: self.report.get_all_datasources_query_failed = True logger.info(f"Get all datasources query failed due to error {e}") @@ -763,7 +762,7 @@ def get_connection_objects( offset += count - for obj in connection_objects.get(tableau_constant.NODES, []): + for obj in connection_objects.get(tableau_constant.NODES) or []: yield obj def emit_workbooks(self) -> Iterable[MetadataWorkUnit]: @@ -977,18 +976,19 @@ def get_upstream_tables(self, tables, datasource_name, browse_path, is_custom_sq ) continue - schema = table.get(tableau_constant.SCHEMA, "") - table_name = table.get(tableau_constant.NAME, "") - full_name = table.get(tableau_constant.FULL_NAME, "") + schema = table.get(tableau_constant.SCHEMA) or "" + table_name = table.get(tableau_constant.NAME) or "" + full_name = table.get(tableau_constant.FULL_NAME) or "" upstream_db = ( - table.get(tableau_constant.DATABASE, {}).get(tableau_constant.NAME, "") - if table.get(tableau_constant.DATABASE) is not None + table[tableau_constant.DATABASE][tableau_constant.NAME] + if table.get(tableau_constant.DATABASE) + and table[tableau_constant.DATABASE].get(tableau_constant.NAME) else "" ) logger.debug( "Processing Table with Connection Type: {0} and id {1}".format( - table.get(tableau_constant.CONNECTION_TYPE, ""), - table.get(tableau_constant.ID, ""), + table.get(tableau_constant.CONNECTION_TYPE) or "", + table.get(tableau_constant.ID) or "", ) ) schema = self._get_schema(schema, upstream_db, full_name) @@ -1008,7 +1008,7 @@ def get_upstream_tables(self, tables, datasource_name, browse_path, is_custom_sq table_urn = make_table_urn( self.config.env, upstream_db, - table.get(tableau_constant.CONNECTION_TYPE, ""), + table.get(tableau_constant.CONNECTION_TYPE) or "", schema, table_name, self.config.platform_instance_map, @@ -1209,7 +1209,9 @@ def emit_custom_sql_datasources(self) -> Iterable[MetadataWorkUnit]: f"Adding datasource {datasource_name}({datasource.get('id')}) to container" ) yield from add_entity_to_container( - self.gen_workbook_key(datasource[tableau_constant.WORKBOOK]), + self.gen_workbook_key( + datasource[tableau_constant.WORKBOOK][tableau_constant.ID] + ), tableau_constant.DATASET, dataset_snapshot.urn, ) @@ -1254,12 +1256,13 @@ def emit_custom_sql_datasources(self) -> Iterable[MetadataWorkUnit]: dataset_snapshot.aspects.append(dataset_properties) - view_properties = ViewPropertiesClass( - materialized=False, - viewLanguage=tableau_constant.SQL, - viewLogic=clean_query(csql.get(tableau_constant.QUERY) or ""), - ) - dataset_snapshot.aspects.append(view_properties) + if csql.get(tableau_constant.QUERY): + view_properties = ViewPropertiesClass( + materialized=False, + viewLanguage=tableau_constant.SQL, + viewLogic=clean_query(csql[tableau_constant.QUERY]), + ) + dataset_snapshot.aspects.append(view_properties) yield self.get_metadata_change_event(dataset_snapshot) yield self.get_metadata_change_proposal( @@ -1292,7 +1295,7 @@ def get_schema_metadata_for_custom_sql( fieldPath=field[tableau_constant.NAME], type=SchemaFieldDataType(type=TypeClass()), nativeDataType=nativeDataType, - description=field.get(tableau_constant.DESCRIPTION, ""), + description=field.get(tableau_constant.DESCRIPTION), ) fields.append(schema_field) @@ -1314,8 +1317,12 @@ def _get_published_datasource_project_luid(self, ds): and ds[tableau_constant.LUID] not in self.datasource_project_map.keys() and self.report.get_all_datasources_query_failed ): + logger.debug( + f"published datasource {ds.get(tableau_constant.NAME)} project_luid not found." + f" Running get datasource query for {ds[tableau_constant.LUID]}" + ) # Query and update self.datasource_project_map with luid - self._query_published_datasource_for_project_luid(ds) + self._query_published_datasource_for_project_luid(ds[tableau_constant.LUID]) if ( ds.get(tableau_constant.LUID) @@ -1331,16 +1338,12 @@ def _get_published_datasource_project_luid(self, ds): return None - def _query_published_datasource_for_project_luid(self, ds: dict) -> None: + def _query_published_datasource_for_project_luid(self, ds_luid: str) -> None: if self.server is None: return try: - logger.debug( - f"published datasource {ds.get(tableau_constant.NAME)} project_luid not found." - f" Running get datasource query for {ds[tableau_constant.LUID]}" - ) - ds_result = self.server.datasources.get_by_id(ds[tableau_constant.LUID]) + ds_result = self.server.datasources.get_by_id(ds_luid) if ds_result.project_id not in self.tableau_project_registry: logger.debug( f"project id ({ds_result.project_id}) of datasource {ds_result.name} is not present in project " @@ -1351,7 +1354,7 @@ def _query_published_datasource_for_project_luid(self, ds: dict) -> None: except Exception as e: self.report.num_get_datasource_query_failures += 1 logger.warning( - f"Failed to get datasource project_luid for {ds[tableau_constant.LUID]} due to error {e}" + f"Failed to get datasource project_luid for {ds_luid} due to error {e}" ) logger.debug("Error stack trace", exc_info=True) @@ -1589,11 +1592,11 @@ def emit_datasource( # Ownership owner = ( self._get_ownership( - datasource_info.get(tableau_constant.OWNER, {}).get( - tableau_constant.USERNAME, "" - ) + datasource_info[tableau_constant.OWNER][tableau_constant.USERNAME] ) if datasource_info + and datasource_info.get(tableau_constant.OWNER) + and datasource_info[tableau_constant.OWNER].get(tableau_constant.USERNAME) else None ) if owner is not None: @@ -1603,24 +1606,15 @@ def emit_datasource( dataset_props = DatasetPropertiesClass( name=datasource.get(tableau_constant.NAME), description=datasource.get(tableau_constant.DESCRIPTION), - customProperties={ - tableau_constant.HAS_EXTRACTS: str( - datasource.get(tableau_constant.HAS_EXTRACTS, "") - ), - tableau_constant.EXTRACT_LAST_REFRESH_TIME: datasource.get( - tableau_constant.EXTRACT_LAST_REFRESH_TIME, "" - ) - or "", - tableau_constant.EXTRACT_LAST_INCREMENTAL_UPDATE_TIME: datasource.get( - tableau_constant.EXTRACT_LAST_INCREMENTAL_UPDATE_TIME, "" - ) - or "", - tableau_constant.EXTRACT_LAST_UPDATE_TIME: datasource.get( - tableau_constant.EXTRACT_LAST_UPDATE_TIME, "" - ) - or "", - tableau_constant.TYPE: datasource.get(tableau_constant.TYPE_NAME, ""), - }, + customProperties=self.get_custom_props_from_dict( + datasource, + [ + tableau_constant.HAS_EXTRACTS, + tableau_constant.EXTRACT_LAST_REFRESH_TIME, + tableau_constant.EXTRACT_LAST_INCREMENTAL_UPDATE_TIME, + tableau_constant.EXTRACT_LAST_UPDATE_TIME, + ], + ), ) dataset_snapshot.aspects.append(dataset_props) @@ -1681,11 +1675,14 @@ def emit_datasource( dataset_snapshot.urn, ) + def get_custom_props_from_dict(self, obj: dict, keys: List[str]) -> Optional[dict]: + return {key: str(obj[key]) for key in keys if obj.get(key)} or None + def _get_datasource_container_key(self, datasource, workbook, is_embedded_ds): container_key: Optional[ContainerKey] = None if is_embedded_ds: # It is embedded then parent is container is workbook if workbook is not None: - container_key = self.gen_workbook_key(workbook) + container_key = self.gen_workbook_key(workbook[tableau_constant.ID]) else: logger.warning( f"Parent container not set for embedded datasource {datasource[tableau_constant.ID]}" @@ -1784,7 +1781,7 @@ def get_schema_metadata_for_table( schema_field = SchemaField( fieldPath=field[tableau_constant.NAME], type=SchemaFieldDataType(type=TypeClass()), - description="", + description=field.get(tableau_constant.DESCRIPTION), nativeDataType=nativeDataType, ) @@ -1804,7 +1801,7 @@ def get_schema_metadata_for_table( def get_sheetwise_upstream_datasources(self, sheet: dict) -> set: sheet_upstream_datasources = set() - for field in sheet.get(tableau_constant.DATA_SOURCE_FIELDS, ""): + for field in sheet.get(tableau_constant.DATA_SOURCE_FIELDS) or []: if field and field.get(tableau_constant.DATA_SOURCE): sheet_upstream_datasources.add( field[tableau_constant.DATA_SOURCE][tableau_constant.ID] @@ -1891,12 +1888,15 @@ def emit_sheets_as_charts( sheet.get(tableau_constant.CONTAINED_IN_DASHBOARDS) is not None and len(sheet[tableau_constant.CONTAINED_IN_DASHBOARDS]) > 0 and sheet[tableau_constant.CONTAINED_IN_DASHBOARDS][0] is not None + and sheet[tableau_constant.CONTAINED_IN_DASHBOARDS][0].get( + tableau_constant.PATH + ) ): # sheet contained in dashboard site_part = f"/t/{self.config.site}" if self.config.site else "" - dashboard_path = sheet[tableau_constant.CONTAINED_IN_DASHBOARDS][0].get( - tableau_constant.PATH, "" - ) + dashboard_path = sheet[tableau_constant.CONTAINED_IN_DASHBOARDS][0][ + tableau_constant.PATH + ] sheet_external_url = f"{self.config.connect_uri}{site_part}/authoring/{dashboard_path}/{sheet.get(tableau_constant.NAME, '')}" else: # hidden or viz-in-tooltip sheet @@ -1920,15 +1920,15 @@ def emit_sheets_as_charts( # Chart Info chart_info = ChartInfoClass( description="", - title=sheet.get(tableau_constant.NAME, ""), + title=sheet.get(tableau_constant.NAME) or "", lastModified=last_modified, externalUrl=sheet_external_url if self.config.ingest_external_links_for_charts else None, inputs=sorted(datasource_urn), - customProperties={ - tableau_constant.LUID: sheet.get(tableau_constant.LUID) or "" - }, + customProperties=self.get_custom_props_from_dict( + sheet, [tableau_constant.LUID] + ), ) chart_snapshot.aspects.append(chart_info) # chart_snapshot doesn't support the stat aspect as list element and hence need to emit MCP @@ -1964,13 +1964,10 @@ def emit_sheets_as_charts( chart_snapshot.aspects.append(owner) # Tags - tag_list = sheet.get(tableau_constant.TAGS, []) - if tag_list and self.config.ingest_tags: - tag_list_str = [ - t.get(tableau_constant.NAME, "") for t in tag_list if t is not None - ] + tags = self.get_tags(sheet) + if tags: chart_snapshot.aspects.append( - builder.make_global_tag_aspect_with_tag_list(tag_list_str) + builder.make_global_tag_aspect_with_tag_list(tags) ) yield self.get_metadata_change_event(chart_snapshot) if sheet_external_url is not None and self.config.ingest_embed_url is True: @@ -1982,7 +1979,7 @@ def emit_sheets_as_charts( ) if workbook is not None: yield from add_entity_to_container( - self.gen_workbook_key(workbook), + self.gen_workbook_key(workbook[tableau_constant.ID]), tableau_constant.CHART, chart_snapshot.urn, ) @@ -2037,7 +2034,7 @@ def populate_sheet_upstream_fields( ) def emit_workbook_as_container(self, workbook: Dict) -> Iterable[MetadataWorkUnit]: - workbook_container_key = self.gen_workbook_key(workbook) + workbook_container_key = self.gen_workbook_key(workbook[tableau_constant.ID]) creator = workbook.get(tableau_constant.OWNER, {}).get( tableau_constant.USERNAME ) @@ -2049,11 +2046,9 @@ def emit_workbook_as_container(self, workbook: Dict) -> Iterable[MetadataWorkUni ) site_part = f"/site/{self.config.site}" if self.config.site else "" - workbook_uri = workbook.get("uri", "") + workbook_uri = workbook.get("uri") workbook_part = ( - workbook_uri[workbook_uri.index("/workbooks/") :] - if workbook.get("uri") - else None + workbook_uri[workbook_uri.index("/workbooks/") :] if workbook_uri else None ) workbook_external_url = ( f"{self.config.connect_uri}/#{site_part}{workbook_part}" @@ -2061,12 +2056,8 @@ def emit_workbook_as_container(self, workbook: Dict) -> Iterable[MetadataWorkUni else None ) - tag_list = workbook.get(tableau_constant.TAGS, []) - tag_list_str = ( - [t.get(tableau_constant.NAME, "") for t in tag_list if t is not None] - if (tag_list and self.config.ingest_tags) - else None - ) + tags = self.get_tags(workbook) + parent_key = None project_luid: Optional[str] = self._get_workbook_project_luid(workbook) if project_luid and project_luid in self.tableau_project_registry.keys(): @@ -2080,20 +2071,20 @@ def emit_workbook_as_container(self, workbook: Dict) -> Iterable[MetadataWorkUni yield from gen_containers( container_key=workbook_container_key, - name=workbook.get(tableau_constant.NAME, ""), + name=workbook.get(tableau_constant.NAME) or "", parent_container_key=parent_key, description=workbook.get(tableau_constant.DESCRIPTION), sub_types=[BIContainerSubTypes.TABLEAU_WORKBOOK], owner_urn=owner_urn, external_url=workbook_external_url, - tags=tag_list_str, + tags=tags, ) - def gen_workbook_key(self, workbook: Dict) -> WorkbookKey: + def gen_workbook_key(self, workbook_id: str) -> WorkbookKey: return WorkbookKey( platform=self.platform, instance=self.config.platform_instance, - workbook_id=workbook[tableau_constant.ID], + workbook_id=workbook_id, ) def gen_project_key(self, project_luid): @@ -2182,6 +2173,18 @@ def emit_dashboards(self) -> Iterable[MetadataWorkUnit]: dashboard, dashboard.get(tableau_constant.WORKBOOK) ) + def get_tags(self, obj: dict) -> Optional[List[str]]: + tag_list = obj.get(tableau_constant.TAGS, []) + if tag_list and self.config.ingest_tags: + tag_list_str = [ + t[tableau_constant.NAME] + for t in tag_list + if t is not None and t.get(tableau_constant.NAME) + ] + + return tag_list_str + return None + def emit_dashboard( self, dashboard: dict, workbook: Optional[Dict] ) -> Iterable[MetadataWorkUnit]: @@ -2223,19 +2226,16 @@ def emit_dashboard( dashboardUrl=dashboard_external_url if self.config.ingest_external_links_for_dashboards else None, - customProperties={ - tableau_constant.LUID: dashboard.get(tableau_constant.LUID) or "" - }, + customProperties=self.get_custom_props_from_dict( + dashboard, [tableau_constant.LUID] + ), ) dashboard_snapshot.aspects.append(dashboard_info_class) - tag_list = dashboard.get(tableau_constant.TAGS, []) - if tag_list and self.config.ingest_tags: - tag_list_str = [ - t.get(tableau_constant.NAME, "") for t in tag_list if t is not None - ] + tags = self.get_tags(dashboard) + if tags: dashboard_snapshot.aspects.append( - builder.make_global_tag_aspect_with_tag_list(tag_list_str) + builder.make_global_tag_aspect_with_tag_list(tags) ) if self.config.extract_usage_stats: @@ -2293,7 +2293,7 @@ def emit_dashboard( if workbook is not None: yield from add_entity_to_container( - self.gen_workbook_key(workbook), + self.gen_workbook_key(workbook[tableau_constant.ID]), tableau_constant.DASHBOARD, dashboard_snapshot.urn, ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau_constant.py b/metadata-ingestion/src/datahub/ingestion/source/tableau_constant.py index da398df932841..e80c9d8fd1f25 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau_constant.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau_constant.py @@ -63,7 +63,6 @@ EXTRACT_LAST_REFRESH_TIME = "extractLastRefreshTime" EXTRACT_LAST_INCREMENTAL_UPDATE_TIME = "extractLastIncrementalUpdateTime" EXTRACT_LAST_UPDATE_TIME = "extractLastUpdateTime" -TYPE = "type" PUBLISHED_DATA_SOURCES_CONNECTION = "publishedDatasourcesConnection" DATA_SOURCE_FIELDS = "datasourceFields" SHEETS_CONNECTION = "sheetsConnection" diff --git a/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json index 665541a140cc9..e6782c1c9f5f3 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json @@ -1308,9 +1308,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign List", "title": "Campaign List", "description": "", @@ -1869,9 +1867,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Summary", "title": "Summary", "description": "", @@ -2482,9 +2478,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile - Sent by Campaign", "title": "Mobile - Sent by Campaign", "description": "", @@ -2991,9 +2985,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet1", "title": "Sheet 1", "description": "", @@ -3136,9 +3128,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet2", "title": "Sheet 2", "description": "", @@ -3449,9 +3439,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet3", "title": "Sheet 3", "description": "", @@ -3646,9 +3634,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Requests", "title": "Opened Requests", "description": "", @@ -4147,9 +4133,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top 10 Items by Requests and YoY Change", "title": "Top 10 Items by Requests and YoY Change", "description": "", @@ -4616,9 +4600,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Problems", "title": "Opened Problems", "description": "", @@ -5117,9 +5099,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue", "title": "Overdue", "description": "", @@ -5499,9 +5479,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High and Critical Priority Problems", "title": "High and Critical Priority Problems", "description": "", @@ -5861,9 +5839,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total Incidents by Category and YoY Change", "title": "Total Incidents by Category and YoY Change", "description": "", @@ -6246,9 +6222,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known Errors", "title": "Known Errors", "description": "", @@ -6686,9 +6660,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Requests", "title": "Overdue Requests", "description": "", @@ -7100,9 +7072,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG Time to Solve an Incident", "title": "AVG Time to Solve an Incident", "description": "", @@ -7430,9 +7400,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made SLA?", "title": "Made SLA?", "description": "", @@ -7792,9 +7760,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-IncidentBreakdownbyPriority", "title": "Tooltip - Incident Breakdown by Priority", "description": "", @@ -8044,9 +8010,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Problems", "title": "Overdue Problems", "description": "", @@ -8484,9 +8448,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-ProblemBreakdownbyPriority", "title": "Tooltip - Problem Breakdown by Priority", "description": "", @@ -8820,9 +8782,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-RequestBreakdownbyPriority", "title": "Tooltip - Request Breakdown by Priority", "description": "", @@ -9182,9 +9142,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age of Active Problems", "title": "Age of Active Problems", "description": "", @@ -9570,9 +9528,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Incidents", "title": "Opened Incidents", "description": "", @@ -10010,9 +9966,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/Workbookpublishedds/Sheet1", "title": "published sheet ds", "description": "", @@ -10323,9 +10277,7 @@ }, { "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "title": "dvd Rental Dashboard", "description": "", "charts": [ @@ -10425,9 +10377,7 @@ }, { "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "title": "Story 1", "description": "", "charts": [], @@ -10525,9 +10475,7 @@ }, { "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "title": "Executive Dashboard", "description": "", "charts": [ @@ -11006,9 +10954,7 @@ "customProperties": { "hasExtracts": "True", "extractLastRefreshTime": "2018-02-09T00:05:25Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-02-09T00:05:25Z", - "type": "EmbeddedDatasource" + "extractLastUpdateTime": "2018-02-09T00:05:25Z" }, "name": "Marketo", "tags": [] @@ -12943,13 +12889,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, + "customProperties": {}, "name": "Customer Payment Query", "tags": [] } @@ -13279,13 +13219,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, + "customProperties": {}, "name": "actor+ (dvdrental)", "tags": [] } @@ -14189,13 +14123,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, + "customProperties": {}, "name": "Superstore Datasource", "tags": [] } @@ -15148,9 +15076,7 @@ "customProperties": { "hasExtracts": "True", "extractLastRefreshTime": "2018-01-18T19:35:39Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-01-18T19:35:39Z", - "type": "EmbeddedDatasource" + "extractLastUpdateTime": "2018-01-18T19:35:39Z" }, "name": "Requests", "tags": [] @@ -22011,9 +21937,7 @@ "customProperties": { "hasExtracts": "True", "extractLastRefreshTime": "2018-01-18T20:21:33Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-01-18T20:21:33Z", - "type": "EmbeddedDatasource" + "extractLastUpdateTime": "2018-01-18T20:21:33Z" }, "name": "Problems", "tags": [] @@ -26122,9 +26046,7 @@ "customProperties": { "hasExtracts": "True", "extractLastRefreshTime": "2018-01-18T20:13:08Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-01-18T20:13:08Z", - "type": "EmbeddedDatasource" + "extractLastUpdateTime": "2018-01-18T20:13:08Z" }, "name": "Incidents", "tags": [] @@ -31464,13 +31386,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, + "customProperties": {}, "name": "test publish datasource", "tags": [] } @@ -31858,13 +31774,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "PublishedDatasource" - }, + "customProperties": {}, "name": "test publish datasource", "description": "description for test publish datasource", "tags": [] @@ -32324,13 +32234,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "PublishedDatasource" - }, + "customProperties": {}, "name": "Superstore Datasource", "description": "Description for Superstore dataset", "tags": [] @@ -33712,7 +33616,6 @@ { "fieldPath": "Test_Variant", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33725,7 +33628,6 @@ { "fieldPath": "Mailing_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33738,7 +33640,6 @@ { "fieldPath": "Campaign_Run_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33751,7 +33652,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33764,7 +33664,6 @@ { "fieldPath": "Activity_Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -33777,7 +33676,6 @@ { "fieldPath": "Choice_Number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33790,7 +33688,6 @@ { "fieldPath": "Step_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33803,7 +33700,6 @@ { "fieldPath": "Campaign_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33816,7 +33712,6 @@ { "fieldPath": "Lead_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33829,7 +33724,6 @@ { "fieldPath": "Has_Predictive", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -33910,7 +33804,6 @@ { "fieldPath": "Campaign_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33923,7 +33816,6 @@ { "fieldPath": "Campaign_Run_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33936,7 +33828,6 @@ { "fieldPath": "Link", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -33949,7 +33840,6 @@ { "fieldPath": "Test_Variant", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33962,7 +33852,6 @@ { "fieldPath": "Platform", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -33975,7 +33864,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33988,7 +33876,6 @@ { "fieldPath": "Activity_Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34001,7 +33888,6 @@ { "fieldPath": "Choice_Number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34014,7 +33900,6 @@ { "fieldPath": "Mailing_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34027,7 +33912,6 @@ { "fieldPath": "Step_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34040,7 +33924,6 @@ { "fieldPath": "Lead_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34053,7 +33936,6 @@ { "fieldPath": "Link_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34066,7 +33948,6 @@ { "fieldPath": "Is_Predictive", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34079,7 +33960,6 @@ { "fieldPath": "Device", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34092,7 +33972,6 @@ { "fieldPath": "Is_Mobile_Device", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34105,7 +33984,6 @@ { "fieldPath": "User_Agent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34186,7 +34064,6 @@ { "fieldPath": "Platform", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34199,7 +34076,6 @@ { "fieldPath": "Device", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34212,7 +34088,6 @@ { "fieldPath": "Choice_Number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34225,7 +34100,6 @@ { "fieldPath": "Lead_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34238,7 +34112,6 @@ { "fieldPath": "Activity_Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34251,7 +34124,6 @@ { "fieldPath": "Test_Variant", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34264,7 +34136,6 @@ { "fieldPath": "Is_Mobile_Device", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34277,7 +34148,6 @@ { "fieldPath": "Has_Predictive", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34290,7 +34160,6 @@ { "fieldPath": "Step_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34303,7 +34172,6 @@ { "fieldPath": "User_Agent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34316,7 +34184,6 @@ { "fieldPath": "Mailing_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34329,7 +34196,6 @@ { "fieldPath": "Campaign_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34342,7 +34208,6 @@ { "fieldPath": "Campaign_Run_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34355,7 +34220,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34436,7 +34300,6 @@ { "fieldPath": "Test_Variant", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34449,7 +34312,6 @@ { "fieldPath": "Campaign_Run_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34462,7 +34324,6 @@ { "fieldPath": "Activity_Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34475,7 +34336,6 @@ { "fieldPath": "Mailing_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34488,7 +34348,6 @@ { "fieldPath": "Has_Predictive", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34501,7 +34360,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34514,7 +34372,6 @@ { "fieldPath": "Campaign_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34527,7 +34384,6 @@ { "fieldPath": "Step_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34540,7 +34396,6 @@ { "fieldPath": "Lead_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34553,7 +34408,6 @@ { "fieldPath": "Choice_Number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34634,7 +34488,6 @@ { "fieldPath": "programName", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34647,7 +34500,6 @@ { "fieldPath": "name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34660,7 +34512,6 @@ { "fieldPath": "programId", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34673,7 +34524,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34686,7 +34536,6 @@ { "fieldPath": "createdAt", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34699,7 +34548,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34712,7 +34560,6 @@ { "fieldPath": "workspaceName", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34725,7 +34572,6 @@ { "fieldPath": "updatedAt", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34738,7 +34584,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34819,7 +34664,6 @@ { "fieldPath": "postal_code", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34832,7 +34676,6 @@ { "fieldPath": "last_update", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34845,7 +34688,6 @@ { "fieldPath": "phone", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34858,7 +34700,6 @@ { "fieldPath": "address2", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34871,7 +34712,6 @@ { "fieldPath": "address", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34884,7 +34724,6 @@ { "fieldPath": "city_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34897,7 +34736,6 @@ { "fieldPath": "district", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34910,7 +34748,6 @@ { "fieldPath": "address_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34991,7 +34828,6 @@ { "fieldPath": "last_update", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35004,7 +34840,6 @@ { "fieldPath": "last_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35017,7 +34852,6 @@ { "fieldPath": "first_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35030,7 +34864,6 @@ { "fieldPath": "actor_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35111,7 +34944,6 @@ { "fieldPath": "Person", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35124,7 +34956,6 @@ { "fieldPath": "Region", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35202,7 +35033,6 @@ { "fieldPath": "Returned", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35215,7 +35045,6 @@ { "fieldPath": "Order ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35293,7 +35122,6 @@ { "fieldPath": "Product ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35306,7 +35134,6 @@ { "fieldPath": "Category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35319,7 +35146,6 @@ { "fieldPath": "Postal Code", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35332,7 +35158,6 @@ { "fieldPath": "City", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35345,7 +35170,6 @@ { "fieldPath": "Quantity", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35358,7 +35182,6 @@ { "fieldPath": "State", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35371,7 +35194,6 @@ { "fieldPath": "Order Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -35384,7 +35206,6 @@ { "fieldPath": "Customer Name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35397,7 +35218,6 @@ { "fieldPath": "Country/Region", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35410,7 +35230,6 @@ { "fieldPath": "Sales", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35423,7 +35242,6 @@ { "fieldPath": "Segment", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35436,7 +35254,6 @@ { "fieldPath": "Sub-Category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35449,7 +35266,6 @@ { "fieldPath": "Profit", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35462,7 +35278,6 @@ { "fieldPath": "Product Name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35475,7 +35290,6 @@ { "fieldPath": "Customer ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35488,7 +35302,6 @@ { "fieldPath": "Order ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35501,7 +35314,6 @@ { "fieldPath": "Row ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35514,7 +35326,6 @@ { "fieldPath": "Discount", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35527,7 +35338,6 @@ { "fieldPath": "Ship Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -35540,7 +35350,6 @@ { "fieldPath": "Ship Mode", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35553,7 +35362,6 @@ { "fieldPath": "Region", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35633,7 +35441,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35646,7 +35453,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35659,7 +35465,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35672,7 +35477,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35685,7 +35489,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35698,7 +35501,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35711,7 +35513,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35724,7 +35525,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35737,7 +35537,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35750,7 +35549,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35763,7 +35561,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35776,7 +35573,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -35789,7 +35585,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35802,7 +35597,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -35815,7 +35609,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35828,7 +35621,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35841,7 +35633,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35854,7 +35645,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35867,7 +35657,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35880,7 +35669,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35893,7 +35681,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35906,7 +35693,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35919,7 +35705,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35932,7 +35717,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35945,7 +35729,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35958,7 +35741,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35971,7 +35753,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35984,7 +35765,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35997,7 +35777,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36010,7 +35789,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36023,7 +35801,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36036,7 +35813,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36049,7 +35825,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36062,7 +35837,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36075,7 +35849,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36088,7 +35861,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36101,7 +35873,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36114,7 +35885,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36127,7 +35897,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36140,7 +35909,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36153,7 +35921,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36166,7 +35933,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36179,7 +35945,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36192,7 +35957,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36205,7 +35969,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36218,7 +35981,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -36231,7 +35993,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36244,7 +36005,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36257,7 +36017,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36270,7 +36029,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36283,7 +36041,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36296,7 +36053,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36309,7 +36065,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36322,7 +36077,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36335,7 +36089,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36348,7 +36101,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36361,7 +36113,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36374,7 +36125,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36387,7 +36137,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36400,7 +36149,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36413,7 +36161,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36426,7 +36173,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36507,7 +36253,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36520,7 +36265,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36533,7 +36277,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36546,7 +36289,6 @@ { "fieldPath": "requested_for", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36559,7 +36301,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36572,7 +36313,6 @@ { "fieldPath": "price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36585,7 +36325,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36598,7 +36337,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36611,7 +36349,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36624,7 +36361,6 @@ { "fieldPath": "delivery_address", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36637,7 +36373,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36650,7 +36385,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36663,7 +36397,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36676,7 +36409,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36689,7 +36421,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36702,7 +36433,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -36715,7 +36445,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36728,7 +36457,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36741,7 +36469,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36754,7 +36481,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36767,7 +36493,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36780,7 +36505,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36793,7 +36517,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36806,7 +36529,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36819,7 +36541,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36832,7 +36553,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36845,7 +36565,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36858,7 +36577,6 @@ { "fieldPath": "requested_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -36871,7 +36589,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36884,7 +36601,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36897,7 +36613,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36910,7 +36625,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36923,7 +36637,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36936,7 +36649,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36949,7 +36661,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36962,7 +36673,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36975,7 +36685,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36988,7 +36697,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37001,7 +36709,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37014,7 +36721,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37027,7 +36733,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37040,7 +36745,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37053,7 +36757,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37066,7 +36769,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37079,7 +36781,6 @@ { "fieldPath": "request_state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37092,7 +36793,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37105,7 +36805,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37118,7 +36817,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37131,7 +36829,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37144,7 +36841,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37157,7 +36853,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37170,7 +36865,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37183,7 +36877,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37196,7 +36889,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37209,7 +36901,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37222,7 +36913,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37235,7 +36925,6 @@ { "fieldPath": "stage", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37248,7 +36937,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37261,7 +36949,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37274,7 +36961,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37287,7 +36973,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37300,7 +36985,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37313,7 +36997,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37326,7 +37009,6 @@ { "fieldPath": "special_instructions", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37339,7 +37021,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37352,7 +37033,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37365,7 +37045,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37378,7 +37057,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37391,7 +37069,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37404,7 +37081,6 @@ { "fieldPath": "calendar_stc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37485,7 +37161,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37498,7 +37173,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37511,7 +37185,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37524,7 +37197,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37537,7 +37209,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37550,7 +37221,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37563,7 +37233,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37576,7 +37245,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37589,7 +37257,6 @@ { "fieldPath": "configuration_item", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37602,7 +37269,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37615,7 +37281,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37628,7 +37293,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37641,7 +37305,6 @@ { "fieldPath": "recurring_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37654,7 +37317,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37667,7 +37329,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37680,7 +37341,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37693,7 +37353,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37706,7 +37365,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37719,7 +37377,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37732,7 +37389,6 @@ { "fieldPath": "sc_catalog", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37745,7 +37401,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37758,7 +37413,6 @@ { "fieldPath": "stage", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37771,7 +37425,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37784,7 +37437,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37797,7 +37449,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37810,7 +37461,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37823,7 +37473,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37836,7 +37485,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37849,7 +37497,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37862,7 +37509,6 @@ { "fieldPath": "price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37875,7 +37521,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37888,7 +37533,6 @@ { "fieldPath": "estimated_delivery", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37901,7 +37545,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37914,7 +37557,6 @@ { "fieldPath": "context", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37927,7 +37569,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37940,7 +37581,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37953,7 +37593,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37966,7 +37605,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37979,7 +37617,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37992,7 +37629,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38005,7 +37641,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38018,7 +37653,6 @@ { "fieldPath": "billable", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38031,7 +37665,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38044,7 +37677,6 @@ { "fieldPath": "recurring_frequency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38057,7 +37689,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38070,7 +37701,6 @@ { "fieldPath": "cat_item", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38083,7 +37713,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38096,7 +37725,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38109,7 +37737,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38122,7 +37749,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38135,7 +37761,6 @@ { "fieldPath": "order_guide", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38148,7 +37773,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38161,7 +37785,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38174,7 +37797,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38187,7 +37809,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38200,7 +37821,6 @@ { "fieldPath": "backordered", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38213,7 +37833,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38226,7 +37845,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38239,7 +37857,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38252,7 +37869,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38265,7 +37881,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38278,7 +37893,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38291,7 +37905,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38304,7 +37917,6 @@ { "fieldPath": "request", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38317,7 +37929,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38330,7 +37941,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38343,7 +37953,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38356,7 +37965,6 @@ { "fieldPath": "quantity", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38369,7 +37977,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38382,7 +37989,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38395,7 +38001,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38408,7 +38013,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38421,7 +38025,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38434,7 +38037,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38447,7 +38049,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38460,7 +38061,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38541,7 +38141,6 @@ { "fieldPath": "sc_catalogs", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38554,7 +38153,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38567,7 +38165,6 @@ { "fieldPath": "type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38580,7 +38177,6 @@ { "fieldPath": "mobile_picture_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38593,7 +38189,6 @@ { "fieldPath": "workflow", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38606,7 +38201,6 @@ { "fieldPath": "sys_customer_update", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38619,7 +38213,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38632,7 +38225,6 @@ { "fieldPath": "visible_standalone", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38645,7 +38237,6 @@ { "fieldPath": "no_quantity", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38658,7 +38249,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38671,7 +38261,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38684,7 +38273,6 @@ { "fieldPath": "sys_scope", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38697,7 +38285,6 @@ { "fieldPath": "template", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38710,7 +38297,6 @@ { "fieldPath": "no_proceed_checkout", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38723,7 +38309,6 @@ { "fieldPath": "billable", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38736,7 +38321,6 @@ { "fieldPath": "name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38749,7 +38333,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38762,7 +38345,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38775,7 +38357,6 @@ { "fieldPath": "meta", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38788,7 +38369,6 @@ { "fieldPath": "ordered_item_link", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38801,7 +38381,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38814,7 +38393,6 @@ { "fieldPath": "sc_ic_version", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38827,7 +38405,6 @@ { "fieldPath": "image", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38840,7 +38417,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38853,7 +38429,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38866,7 +38441,6 @@ { "fieldPath": "sys_policy", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38879,7 +38453,6 @@ { "fieldPath": "roles", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38892,7 +38465,6 @@ { "fieldPath": "picture", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38905,7 +38477,6 @@ { "fieldPath": "list_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38918,7 +38489,6 @@ { "fieldPath": "no_order_now", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38931,7 +38501,6 @@ { "fieldPath": "vendor", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38944,7 +38513,6 @@ { "fieldPath": "sc_ic_item_staging", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38957,7 +38525,6 @@ { "fieldPath": "no_order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38970,7 +38537,6 @@ { "fieldPath": "entitlement_script", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38983,7 +38549,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38996,7 +38561,6 @@ { "fieldPath": "icon", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39009,7 +38573,6 @@ { "fieldPath": "ignore_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39022,7 +38585,6 @@ { "fieldPath": "start_closed", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39035,7 +38597,6 @@ { "fieldPath": "sys_package", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39048,7 +38609,6 @@ { "fieldPath": "group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39061,7 +38621,6 @@ { "fieldPath": "sys_replace_on_upgrade", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39074,7 +38633,6 @@ { "fieldPath": "cost", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39087,7 +38645,6 @@ { "fieldPath": "use_sc_layout", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39100,7 +38657,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39113,7 +38669,6 @@ { "fieldPath": "availability", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39126,7 +38681,6 @@ { "fieldPath": "model", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39139,7 +38693,6 @@ { "fieldPath": "custom_cart", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39152,7 +38705,6 @@ { "fieldPath": "mobile_picture", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39165,7 +38717,6 @@ { "fieldPath": "category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39178,7 +38729,6 @@ { "fieldPath": "no_cart", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39191,7 +38741,6 @@ { "fieldPath": "mobile_hide_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39204,7 +38753,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39217,7 +38765,6 @@ { "fieldPath": "price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39230,7 +38777,6 @@ { "fieldPath": "delivery_time", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39243,7 +38789,6 @@ { "fieldPath": "no_search", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39256,7 +38801,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39269,7 +38813,6 @@ { "fieldPath": "recurring_frequency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39282,7 +38825,6 @@ { "fieldPath": "recurring_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39295,7 +38837,6 @@ { "fieldPath": "delivery_plan_script", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39308,7 +38849,6 @@ { "fieldPath": "visible_bundle", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39321,7 +38861,6 @@ { "fieldPath": "sys_update_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39334,7 +38873,6 @@ { "fieldPath": "sys_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39347,7 +38885,6 @@ { "fieldPath": "visible_guide", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39360,7 +38897,6 @@ { "fieldPath": "preview", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39373,7 +38909,6 @@ { "fieldPath": "omit_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39454,7 +38989,6 @@ { "fieldPath": "u_u", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39467,7 +39001,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39480,7 +39013,6 @@ { "fieldPath": "source", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39493,7 +39025,6 @@ { "fieldPath": "exclude_manager", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39506,7 +39037,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39519,7 +39049,6 @@ { "fieldPath": "cost_center", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39532,7 +39061,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39545,7 +39073,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39558,7 +39085,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39571,7 +39097,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39584,7 +39109,6 @@ { "fieldPath": "u_lucha", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39597,7 +39121,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39610,7 +39133,6 @@ { "fieldPath": "u_lu2", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39623,7 +39145,6 @@ { "fieldPath": "type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39636,7 +39157,6 @@ { "fieldPath": "roles", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39649,7 +39169,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39662,7 +39181,6 @@ { "fieldPath": "default_assignee", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39675,7 +39193,6 @@ { "fieldPath": "email", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39688,7 +39205,6 @@ { "fieldPath": "manager", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39701,7 +39217,6 @@ { "fieldPath": "name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39714,7 +39229,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39727,7 +39241,6 @@ { "fieldPath": "include_members", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39808,7 +39321,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39821,7 +39333,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39834,7 +39345,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39847,7 +39357,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39860,7 +39369,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39873,7 +39381,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39886,7 +39393,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39899,7 +39405,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39912,7 +39417,6 @@ { "fieldPath": "related_incidents", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39925,7 +39429,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39938,7 +39441,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39951,7 +39453,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39964,7 +39465,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39977,7 +39477,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39990,7 +39489,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40003,7 +39501,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40016,7 +39513,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40029,7 +39525,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40042,7 +39537,6 @@ { "fieldPath": "rfc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40055,7 +39549,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40068,7 +39561,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40081,7 +39573,6 @@ { "fieldPath": "problem_state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40094,7 +39585,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40107,7 +39597,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40120,7 +39609,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -40133,7 +39621,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40146,7 +39633,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40159,7 +39645,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40172,7 +39657,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40185,7 +39669,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40198,7 +39681,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40211,7 +39693,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -40224,7 +39705,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40237,7 +39717,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40250,7 +39729,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40263,7 +39741,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40276,7 +39753,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40289,7 +39765,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40302,7 +39777,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40315,7 +39789,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40328,7 +39801,6 @@ { "fieldPath": "work_around", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40341,7 +39813,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40354,7 +39825,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40367,7 +39837,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40380,7 +39849,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40393,7 +39861,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40406,7 +39873,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40419,7 +39885,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40432,7 +39897,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40445,7 +39909,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40458,7 +39921,6 @@ { "fieldPath": "known_error", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -40471,7 +39933,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40484,7 +39945,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40497,7 +39957,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40510,7 +39969,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40523,7 +39981,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40536,7 +39993,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40549,7 +40005,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40562,7 +40017,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40575,7 +40029,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40588,7 +40041,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40601,7 +40053,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40614,7 +40065,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40627,7 +40077,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40640,7 +40089,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40653,7 +40101,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40666,7 +40113,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40747,7 +40193,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40760,7 +40205,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40773,7 +40217,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40786,7 +40229,6 @@ { "fieldPath": "severity", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40799,7 +40241,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40812,7 +40253,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40825,7 +40265,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40838,7 +40277,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40851,7 +40289,6 @@ { "fieldPath": "parent_incident", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40864,7 +40301,6 @@ { "fieldPath": "subcategory", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40877,7 +40313,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40890,7 +40325,6 @@ { "fieldPath": "caller_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40903,7 +40337,6 @@ { "fieldPath": "resolved_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40916,7 +40349,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40929,7 +40361,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40942,7 +40373,6 @@ { "fieldPath": "resolved_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40955,7 +40385,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40968,7 +40397,6 @@ { "fieldPath": "business_stc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40981,7 +40409,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40994,7 +40421,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41007,7 +40433,6 @@ { "fieldPath": "category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41020,7 +40445,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41033,7 +40457,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41046,7 +40469,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41059,7 +40481,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41072,7 +40493,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41085,7 +40505,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41098,7 +40517,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41111,7 +40529,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41124,7 +40541,6 @@ { "fieldPath": "child_incidents", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41137,7 +40553,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41150,7 +40565,6 @@ { "fieldPath": "incident_state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41163,7 +40577,6 @@ { "fieldPath": "notify", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41176,7 +40589,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41189,7 +40601,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41202,7 +40613,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41215,7 +40625,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41228,7 +40637,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41241,7 +40649,6 @@ { "fieldPath": "problem_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41254,7 +40661,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41267,7 +40673,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41280,7 +40685,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41293,7 +40697,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41306,7 +40709,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41319,7 +40721,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41332,7 +40733,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41345,7 +40745,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41358,7 +40757,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41371,7 +40769,6 @@ { "fieldPath": "caused_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41384,7 +40781,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41397,7 +40793,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41410,7 +40805,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41423,7 +40817,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -41436,7 +40829,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41449,7 +40841,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41462,7 +40853,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41475,7 +40865,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41488,7 +40877,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41501,7 +40889,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41514,7 +40901,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41527,7 +40913,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41540,7 +40925,6 @@ { "fieldPath": "reopen_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41553,7 +40937,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41566,7 +40949,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -41579,7 +40961,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41592,7 +40973,6 @@ { "fieldPath": "calendar_stc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41605,7 +40985,6 @@ { "fieldPath": "rfc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41618,7 +40997,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41631,7 +41009,6 @@ { "fieldPath": "close_code", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41644,7 +41021,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41657,7 +41033,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41670,7 +41045,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41683,7 +41057,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -41696,7 +41069,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41709,7 +41081,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41722,7 +41093,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41735,7 +41105,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41748,7 +41117,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41761,7 +41129,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41842,7 +41209,6 @@ { "fieldPath": "first_discovered", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41855,7 +41221,6 @@ { "fieldPath": "operational_status", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41868,7 +41233,6 @@ { "fieldPath": "last_discovered", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41881,7 +41245,6 @@ { "fieldPath": "cost_cc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41894,7 +41257,6 @@ { "fieldPath": "checked_in", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41907,7 +41269,6 @@ { "fieldPath": "attributes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41920,7 +41281,6 @@ { "fieldPath": "serial_number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41933,7 +41293,6 @@ { "fieldPath": "vendor", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41946,7 +41305,6 @@ { "fieldPath": "ip_address", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41959,7 +41317,6 @@ { "fieldPath": "support_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41972,7 +41329,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41985,7 +41341,6 @@ { "fieldPath": "asset", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41998,7 +41353,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42011,7 +41365,6 @@ { "fieldPath": "supported_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42024,7 +41377,6 @@ { "fieldPath": "invoice_number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42037,7 +41389,6 @@ { "fieldPath": "managed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42050,7 +41401,6 @@ { "fieldPath": "fault_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -42063,7 +41413,6 @@ { "fieldPath": "due_in", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42076,7 +41425,6 @@ { "fieldPath": "cost", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42089,7 +41437,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42102,7 +41449,6 @@ { "fieldPath": "justification", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42115,7 +41461,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42128,7 +41473,6 @@ { "fieldPath": "assigned", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42141,7 +41485,6 @@ { "fieldPath": "model_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42154,7 +41497,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42167,7 +41509,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42180,7 +41521,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42193,7 +41533,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42206,7 +41545,6 @@ { "fieldPath": "lease_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42219,7 +41557,6 @@ { "fieldPath": "monitor", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -42232,7 +41569,6 @@ { "fieldPath": "cost_center", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42245,7 +41581,6 @@ { "fieldPath": "subcategory", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42258,7 +41593,6 @@ { "fieldPath": "delivery_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42271,7 +41605,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42284,7 +41617,6 @@ { "fieldPath": "can_print", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -42297,7 +41629,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42310,7 +41641,6 @@ { "fieldPath": "model_number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42323,7 +41653,6 @@ { "fieldPath": "name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42336,7 +41665,6 @@ { "fieldPath": "start_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42349,7 +41677,6 @@ { "fieldPath": "discovery_source", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42362,7 +41689,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42375,7 +41701,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42388,7 +41713,6 @@ { "fieldPath": "category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42401,7 +41725,6 @@ { "fieldPath": "schedule", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42414,7 +41737,6 @@ { "fieldPath": "fqdn", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42427,7 +41749,6 @@ { "fieldPath": "warranty_expiration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -42440,7 +41761,6 @@ { "fieldPath": "owned_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42453,7 +41773,6 @@ { "fieldPath": "asset_tag", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42466,7 +41785,6 @@ { "fieldPath": "manufacturer", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42479,7 +41797,6 @@ { "fieldPath": "purchase_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -42492,7 +41809,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42505,7 +41821,6 @@ { "fieldPath": "department", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42518,7 +41833,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42531,7 +41845,6 @@ { "fieldPath": "checked_out", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42544,7 +41857,6 @@ { "fieldPath": "unverified", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -42557,7 +41869,6 @@ { "fieldPath": "skip_sync", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -42570,7 +41881,6 @@ { "fieldPath": "po_number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42583,7 +41893,6 @@ { "fieldPath": "order_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42596,7 +41905,6 @@ { "fieldPath": "gl_account", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42609,7 +41917,6 @@ { "fieldPath": "maintenance_schedule", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42622,7 +41929,6 @@ { "fieldPath": "install_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42635,7 +41941,6 @@ { "fieldPath": "dns_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42648,7 +41953,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42661,7 +41965,6 @@ { "fieldPath": "mac_address", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42674,7 +41977,6 @@ { "fieldPath": "change_control", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42687,7 +41989,6 @@ { "fieldPath": "install_status", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -42700,7 +42001,6 @@ { "fieldPath": "due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42713,7 +42013,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} diff --git a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json index 45b1ebca25b8a..ea15a2ac56b58 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json @@ -1076,9 +1076,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign List", "title": "Campaign List", "description": "", @@ -1637,9 +1635,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Summary", "title": "Summary", "description": "", @@ -2250,9 +2246,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile - Sent by Campaign", "title": "Mobile - Sent by Campaign", "description": "", @@ -2759,9 +2753,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet1", "title": "Sheet 1", "description": "", @@ -2904,9 +2896,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet2", "title": "Sheet 2", "description": "", @@ -3217,9 +3207,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet3", "title": "Sheet 3", "description": "", @@ -3414,9 +3402,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Requests", "title": "Opened Requests", "description": "", @@ -3915,9 +3901,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top 10 Items by Requests and YoY Change", "title": "Top 10 Items by Requests and YoY Change", "description": "", @@ -4384,9 +4368,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Problems", "title": "Opened Problems", "description": "", @@ -4885,9 +4867,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue", "title": "Overdue", "description": "", @@ -5267,9 +5247,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High and Critical Priority Problems", "title": "High and Critical Priority Problems", "description": "", @@ -5629,9 +5607,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total Incidents by Category and YoY Change", "title": "Total Incidents by Category and YoY Change", "description": "", @@ -6014,9 +5990,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known Errors", "title": "Known Errors", "description": "", @@ -6454,9 +6428,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Requests", "title": "Overdue Requests", "description": "", @@ -6868,9 +6840,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG Time to Solve an Incident", "title": "AVG Time to Solve an Incident", "description": "", @@ -7198,9 +7168,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made SLA?", "title": "Made SLA?", "description": "", @@ -7560,9 +7528,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-IncidentBreakdownbyPriority", "title": "Tooltip - Incident Breakdown by Priority", "description": "", @@ -7812,9 +7778,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Problems", "title": "Overdue Problems", "description": "", @@ -8252,9 +8216,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-ProblemBreakdownbyPriority", "title": "Tooltip - Problem Breakdown by Priority", "description": "", @@ -8588,9 +8550,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-RequestBreakdownbyPriority", "title": "Tooltip - Request Breakdown by Priority", "description": "", @@ -8950,9 +8910,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age of Active Problems", "title": "Age of Active Problems", "description": "", @@ -9338,9 +9296,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Incidents", "title": "Opened Incidents", "description": "", @@ -9778,9 +9734,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/Workbookpublishedds/Sheet1", "title": "published sheet ds", "description": "", @@ -10091,9 +10045,7 @@ }, { "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "title": "dvd Rental Dashboard", "description": "", "charts": [ @@ -10193,9 +10145,7 @@ }, { "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "title": "Story 1", "description": "", "charts": [], @@ -10293,9 +10243,7 @@ }, { "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "title": "Executive Dashboard", "description": "", "charts": [ @@ -10774,9 +10722,7 @@ "customProperties": { "hasExtracts": "True", "extractLastRefreshTime": "2018-02-09T00:05:25Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-02-09T00:05:25Z", - "type": "EmbeddedDatasource" + "extractLastUpdateTime": "2018-02-09T00:05:25Z" }, "name": "Marketo", "tags": [] @@ -12711,13 +12657,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, + "customProperties": {}, "name": "Customer Payment Query", "tags": [] } @@ -13047,13 +12987,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, + "customProperties": {}, "name": "actor+ (dvdrental)", "tags": [] } @@ -13957,13 +13891,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, + "customProperties": {}, "name": "Superstore Datasource", "tags": [] } @@ -14916,9 +14844,7 @@ "customProperties": { "hasExtracts": "True", "extractLastRefreshTime": "2018-01-18T19:35:39Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-01-18T19:35:39Z", - "type": "EmbeddedDatasource" + "extractLastUpdateTime": "2018-01-18T19:35:39Z" }, "name": "Requests", "tags": [] @@ -21779,9 +21705,7 @@ "customProperties": { "hasExtracts": "True", "extractLastRefreshTime": "2018-01-18T20:21:33Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-01-18T20:21:33Z", - "type": "EmbeddedDatasource" + "extractLastUpdateTime": "2018-01-18T20:21:33Z" }, "name": "Problems", "tags": [] @@ -25890,9 +25814,7 @@ "customProperties": { "hasExtracts": "True", "extractLastRefreshTime": "2018-01-18T20:13:08Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-01-18T20:13:08Z", - "type": "EmbeddedDatasource" + "extractLastUpdateTime": "2018-01-18T20:13:08Z" }, "name": "Incidents", "tags": [] @@ -31232,13 +31154,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, + "customProperties": {}, "name": "test publish datasource", "tags": [] } @@ -31626,13 +31542,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "PublishedDatasource" - }, + "customProperties": {}, "name": "test publish datasource", "description": "description for test publish datasource", "tags": [] @@ -32092,13 +32002,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "PublishedDatasource" - }, + "customProperties": {}, "name": "Superstore Datasource", "description": "Description for Superstore dataset", "tags": [] @@ -33480,7 +33384,6 @@ { "fieldPath": "Test_Variant", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33493,7 +33396,6 @@ { "fieldPath": "Mailing_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33506,7 +33408,6 @@ { "fieldPath": "Campaign_Run_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33519,7 +33420,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33532,7 +33432,6 @@ { "fieldPath": "Activity_Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -33545,7 +33444,6 @@ { "fieldPath": "Choice_Number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33558,7 +33456,6 @@ { "fieldPath": "Step_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33571,7 +33468,6 @@ { "fieldPath": "Campaign_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33584,7 +33480,6 @@ { "fieldPath": "Lead_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33597,7 +33492,6 @@ { "fieldPath": "Has_Predictive", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -33678,7 +33572,6 @@ { "fieldPath": "Campaign_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33691,7 +33584,6 @@ { "fieldPath": "Campaign_Run_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33704,7 +33596,6 @@ { "fieldPath": "Link", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -33717,7 +33608,6 @@ { "fieldPath": "Test_Variant", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33730,7 +33620,6 @@ { "fieldPath": "Platform", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -33743,7 +33632,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33756,7 +33644,6 @@ { "fieldPath": "Activity_Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -33769,7 +33656,6 @@ { "fieldPath": "Choice_Number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33782,7 +33668,6 @@ { "fieldPath": "Mailing_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33795,7 +33680,6 @@ { "fieldPath": "Step_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33808,7 +33692,6 @@ { "fieldPath": "Lead_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33821,7 +33704,6 @@ { "fieldPath": "Link_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -33834,7 +33716,6 @@ { "fieldPath": "Is_Predictive", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -33847,7 +33728,6 @@ { "fieldPath": "Device", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -33860,7 +33740,6 @@ { "fieldPath": "Is_Mobile_Device", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -33873,7 +33752,6 @@ { "fieldPath": "User_Agent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -33954,7 +33832,6 @@ { "fieldPath": "Platform", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -33967,7 +33844,6 @@ { "fieldPath": "Device", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -33980,7 +33856,6 @@ { "fieldPath": "Choice_Number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33993,7 +33868,6 @@ { "fieldPath": "Lead_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34006,7 +33880,6 @@ { "fieldPath": "Activity_Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34019,7 +33892,6 @@ { "fieldPath": "Test_Variant", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34032,7 +33904,6 @@ { "fieldPath": "Is_Mobile_Device", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34045,7 +33916,6 @@ { "fieldPath": "Has_Predictive", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34058,7 +33928,6 @@ { "fieldPath": "Step_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34071,7 +33940,6 @@ { "fieldPath": "User_Agent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34084,7 +33952,6 @@ { "fieldPath": "Mailing_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34097,7 +33964,6 @@ { "fieldPath": "Campaign_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34110,7 +33976,6 @@ { "fieldPath": "Campaign_Run_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34123,7 +33988,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34204,7 +34068,6 @@ { "fieldPath": "Test_Variant", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34217,7 +34080,6 @@ { "fieldPath": "Campaign_Run_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34230,7 +34092,6 @@ { "fieldPath": "Activity_Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34243,7 +34104,6 @@ { "fieldPath": "Mailing_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34256,7 +34116,6 @@ { "fieldPath": "Has_Predictive", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34269,7 +34128,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34282,7 +34140,6 @@ { "fieldPath": "Campaign_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34295,7 +34152,6 @@ { "fieldPath": "Step_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34308,7 +34164,6 @@ { "fieldPath": "Lead_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34321,7 +34176,6 @@ { "fieldPath": "Choice_Number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34402,7 +34256,6 @@ { "fieldPath": "programName", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34415,7 +34268,6 @@ { "fieldPath": "name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34428,7 +34280,6 @@ { "fieldPath": "programId", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34441,7 +34292,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34454,7 +34304,6 @@ { "fieldPath": "createdAt", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34467,7 +34316,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34480,7 +34328,6 @@ { "fieldPath": "workspaceName", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34493,7 +34340,6 @@ { "fieldPath": "updatedAt", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34506,7 +34352,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34587,7 +34432,6 @@ { "fieldPath": "postal_code", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34600,7 +34444,6 @@ { "fieldPath": "last_update", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34613,7 +34456,6 @@ { "fieldPath": "phone", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34626,7 +34468,6 @@ { "fieldPath": "address2", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34639,7 +34480,6 @@ { "fieldPath": "address", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34652,7 +34492,6 @@ { "fieldPath": "city_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34665,7 +34504,6 @@ { "fieldPath": "district", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34678,7 +34516,6 @@ { "fieldPath": "address_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34759,7 +34596,6 @@ { "fieldPath": "last_update", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34772,7 +34608,6 @@ { "fieldPath": "last_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34785,7 +34620,6 @@ { "fieldPath": "first_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34798,7 +34632,6 @@ { "fieldPath": "actor_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34879,7 +34712,6 @@ { "fieldPath": "Person", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34892,7 +34724,6 @@ { "fieldPath": "Region", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34970,7 +34801,6 @@ { "fieldPath": "Returned", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34983,7 +34813,6 @@ { "fieldPath": "Order ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35061,7 +34890,6 @@ { "fieldPath": "Product ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35074,7 +34902,6 @@ { "fieldPath": "Category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35087,7 +34914,6 @@ { "fieldPath": "Postal Code", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35100,7 +34926,6 @@ { "fieldPath": "City", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35113,7 +34938,6 @@ { "fieldPath": "Quantity", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35126,7 +34950,6 @@ { "fieldPath": "State", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35139,7 +34962,6 @@ { "fieldPath": "Order Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -35152,7 +34974,6 @@ { "fieldPath": "Customer Name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35165,7 +34986,6 @@ { "fieldPath": "Country/Region", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35178,7 +34998,6 @@ { "fieldPath": "Sales", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35191,7 +35010,6 @@ { "fieldPath": "Segment", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35204,7 +35022,6 @@ { "fieldPath": "Sub-Category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35217,7 +35034,6 @@ { "fieldPath": "Profit", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35230,7 +35046,6 @@ { "fieldPath": "Product Name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35243,7 +35058,6 @@ { "fieldPath": "Customer ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35256,7 +35070,6 @@ { "fieldPath": "Order ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35269,7 +35082,6 @@ { "fieldPath": "Row ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35282,7 +35094,6 @@ { "fieldPath": "Discount", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35295,7 +35106,6 @@ { "fieldPath": "Ship Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -35308,7 +35118,6 @@ { "fieldPath": "Ship Mode", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35321,7 +35130,6 @@ { "fieldPath": "Region", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35401,7 +35209,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35414,7 +35221,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35427,7 +35233,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35440,7 +35245,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35453,7 +35257,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35466,7 +35269,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35479,7 +35281,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35492,7 +35293,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35505,7 +35305,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35518,7 +35317,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35531,7 +35329,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35544,7 +35341,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -35557,7 +35353,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35570,7 +35365,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -35583,7 +35377,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35596,7 +35389,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35609,7 +35401,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35622,7 +35413,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35635,7 +35425,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35648,7 +35437,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35661,7 +35449,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35674,7 +35461,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35687,7 +35473,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35700,7 +35485,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35713,7 +35497,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35726,7 +35509,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35739,7 +35521,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35752,7 +35533,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35765,7 +35545,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35778,7 +35557,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35791,7 +35569,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35804,7 +35581,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35817,7 +35593,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35830,7 +35605,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35843,7 +35617,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35856,7 +35629,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35869,7 +35641,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35882,7 +35653,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35895,7 +35665,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35908,7 +35677,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35921,7 +35689,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35934,7 +35701,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35947,7 +35713,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35960,7 +35725,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35973,7 +35737,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35986,7 +35749,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -35999,7 +35761,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36012,7 +35773,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36025,7 +35785,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36038,7 +35797,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36051,7 +35809,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36064,7 +35821,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36077,7 +35833,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36090,7 +35845,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36103,7 +35857,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36116,7 +35869,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36129,7 +35881,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36142,7 +35893,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36155,7 +35905,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36168,7 +35917,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36181,7 +35929,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36194,7 +35941,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36275,7 +36021,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36288,7 +36033,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36301,7 +36045,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36314,7 +36057,6 @@ { "fieldPath": "requested_for", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36327,7 +36069,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36340,7 +36081,6 @@ { "fieldPath": "price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36353,7 +36093,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36366,7 +36105,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36379,7 +36117,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36392,7 +36129,6 @@ { "fieldPath": "delivery_address", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36405,7 +36141,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36418,7 +36153,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36431,7 +36165,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36444,7 +36177,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36457,7 +36189,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36470,7 +36201,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -36483,7 +36213,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36496,7 +36225,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36509,7 +36237,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36522,7 +36249,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36535,7 +36261,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36548,7 +36273,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36561,7 +36285,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36574,7 +36297,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36587,7 +36309,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36600,7 +36321,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36613,7 +36333,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36626,7 +36345,6 @@ { "fieldPath": "requested_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -36639,7 +36357,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36652,7 +36369,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36665,7 +36381,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36678,7 +36393,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36691,7 +36405,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36704,7 +36417,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36717,7 +36429,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36730,7 +36441,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36743,7 +36453,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36756,7 +36465,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36769,7 +36477,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36782,7 +36489,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36795,7 +36501,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36808,7 +36513,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36821,7 +36525,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36834,7 +36537,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36847,7 +36549,6 @@ { "fieldPath": "request_state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36860,7 +36561,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36873,7 +36573,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36886,7 +36585,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36899,7 +36597,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -36912,7 +36609,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36925,7 +36621,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36938,7 +36633,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36951,7 +36645,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36964,7 +36657,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36977,7 +36669,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36990,7 +36681,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37003,7 +36693,6 @@ { "fieldPath": "stage", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37016,7 +36705,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37029,7 +36717,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37042,7 +36729,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37055,7 +36741,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37068,7 +36753,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37081,7 +36765,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37094,7 +36777,6 @@ { "fieldPath": "special_instructions", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37107,7 +36789,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37120,7 +36801,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37133,7 +36813,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37146,7 +36825,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37159,7 +36837,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37172,7 +36849,6 @@ { "fieldPath": "calendar_stc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37253,7 +36929,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37266,7 +36941,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37279,7 +36953,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37292,7 +36965,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37305,7 +36977,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37318,7 +36989,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37331,7 +37001,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37344,7 +37013,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37357,7 +37025,6 @@ { "fieldPath": "configuration_item", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37370,7 +37037,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37383,7 +37049,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37396,7 +37061,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37409,7 +37073,6 @@ { "fieldPath": "recurring_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37422,7 +37085,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37435,7 +37097,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37448,7 +37109,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37461,7 +37121,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37474,7 +37133,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37487,7 +37145,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37500,7 +37157,6 @@ { "fieldPath": "sc_catalog", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37513,7 +37169,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37526,7 +37181,6 @@ { "fieldPath": "stage", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37539,7 +37193,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37552,7 +37205,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37565,7 +37217,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37578,7 +37229,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37591,7 +37241,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37604,7 +37253,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37617,7 +37265,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37630,7 +37277,6 @@ { "fieldPath": "price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37643,7 +37289,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37656,7 +37301,6 @@ { "fieldPath": "estimated_delivery", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37669,7 +37313,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37682,7 +37325,6 @@ { "fieldPath": "context", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37695,7 +37337,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37708,7 +37349,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37721,7 +37361,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37734,7 +37373,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37747,7 +37385,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37760,7 +37397,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37773,7 +37409,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37786,7 +37421,6 @@ { "fieldPath": "billable", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37799,7 +37433,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37812,7 +37445,6 @@ { "fieldPath": "recurring_frequency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37825,7 +37457,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37838,7 +37469,6 @@ { "fieldPath": "cat_item", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37851,7 +37481,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37864,7 +37493,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37877,7 +37505,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37890,7 +37517,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37903,7 +37529,6 @@ { "fieldPath": "order_guide", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37916,7 +37541,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37929,7 +37553,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37942,7 +37565,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37955,7 +37577,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37968,7 +37589,6 @@ { "fieldPath": "backordered", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37981,7 +37601,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37994,7 +37613,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38007,7 +37625,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38020,7 +37637,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38033,7 +37649,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38046,7 +37661,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38059,7 +37673,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38072,7 +37685,6 @@ { "fieldPath": "request", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38085,7 +37697,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38098,7 +37709,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38111,7 +37721,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38124,7 +37733,6 @@ { "fieldPath": "quantity", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38137,7 +37745,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38150,7 +37757,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38163,7 +37769,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38176,7 +37781,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38189,7 +37793,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38202,7 +37805,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38215,7 +37817,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38228,7 +37829,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38309,7 +37909,6 @@ { "fieldPath": "sc_catalogs", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38322,7 +37921,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38335,7 +37933,6 @@ { "fieldPath": "type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38348,7 +37945,6 @@ { "fieldPath": "mobile_picture_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38361,7 +37957,6 @@ { "fieldPath": "workflow", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38374,7 +37969,6 @@ { "fieldPath": "sys_customer_update", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38387,7 +37981,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38400,7 +37993,6 @@ { "fieldPath": "visible_standalone", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38413,7 +38005,6 @@ { "fieldPath": "no_quantity", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38426,7 +38017,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38439,7 +38029,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38452,7 +38041,6 @@ { "fieldPath": "sys_scope", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38465,7 +38053,6 @@ { "fieldPath": "template", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38478,7 +38065,6 @@ { "fieldPath": "no_proceed_checkout", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38491,7 +38077,6 @@ { "fieldPath": "billable", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38504,7 +38089,6 @@ { "fieldPath": "name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38517,7 +38101,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38530,7 +38113,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38543,7 +38125,6 @@ { "fieldPath": "meta", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38556,7 +38137,6 @@ { "fieldPath": "ordered_item_link", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38569,7 +38149,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38582,7 +38161,6 @@ { "fieldPath": "sc_ic_version", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38595,7 +38173,6 @@ { "fieldPath": "image", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38608,7 +38185,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38621,7 +38197,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38634,7 +38209,6 @@ { "fieldPath": "sys_policy", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38647,7 +38221,6 @@ { "fieldPath": "roles", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38660,7 +38233,6 @@ { "fieldPath": "picture", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38673,7 +38245,6 @@ { "fieldPath": "list_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38686,7 +38257,6 @@ { "fieldPath": "no_order_now", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38699,7 +38269,6 @@ { "fieldPath": "vendor", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38712,7 +38281,6 @@ { "fieldPath": "sc_ic_item_staging", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38725,7 +38293,6 @@ { "fieldPath": "no_order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38738,7 +38305,6 @@ { "fieldPath": "entitlement_script", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38751,7 +38317,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38764,7 +38329,6 @@ { "fieldPath": "icon", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38777,7 +38341,6 @@ { "fieldPath": "ignore_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38790,7 +38353,6 @@ { "fieldPath": "start_closed", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38803,7 +38365,6 @@ { "fieldPath": "sys_package", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38816,7 +38377,6 @@ { "fieldPath": "group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38829,7 +38389,6 @@ { "fieldPath": "sys_replace_on_upgrade", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38842,7 +38401,6 @@ { "fieldPath": "cost", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38855,7 +38413,6 @@ { "fieldPath": "use_sc_layout", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38868,7 +38425,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38881,7 +38437,6 @@ { "fieldPath": "availability", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38894,7 +38449,6 @@ { "fieldPath": "model", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38907,7 +38461,6 @@ { "fieldPath": "custom_cart", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38920,7 +38473,6 @@ { "fieldPath": "mobile_picture", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38933,7 +38485,6 @@ { "fieldPath": "category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38946,7 +38497,6 @@ { "fieldPath": "no_cart", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38959,7 +38509,6 @@ { "fieldPath": "mobile_hide_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38972,7 +38521,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38985,7 +38533,6 @@ { "fieldPath": "price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38998,7 +38545,6 @@ { "fieldPath": "delivery_time", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39011,7 +38557,6 @@ { "fieldPath": "no_search", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39024,7 +38569,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39037,7 +38581,6 @@ { "fieldPath": "recurring_frequency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39050,7 +38593,6 @@ { "fieldPath": "recurring_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39063,7 +38605,6 @@ { "fieldPath": "delivery_plan_script", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39076,7 +38617,6 @@ { "fieldPath": "visible_bundle", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39089,7 +38629,6 @@ { "fieldPath": "sys_update_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39102,7 +38641,6 @@ { "fieldPath": "sys_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39115,7 +38653,6 @@ { "fieldPath": "visible_guide", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39128,7 +38665,6 @@ { "fieldPath": "preview", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39141,7 +38677,6 @@ { "fieldPath": "omit_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39222,7 +38757,6 @@ { "fieldPath": "u_u", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39235,7 +38769,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39248,7 +38781,6 @@ { "fieldPath": "source", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39261,7 +38793,6 @@ { "fieldPath": "exclude_manager", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39274,7 +38805,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39287,7 +38817,6 @@ { "fieldPath": "cost_center", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39300,7 +38829,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39313,7 +38841,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39326,7 +38853,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39339,7 +38865,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39352,7 +38877,6 @@ { "fieldPath": "u_lucha", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39365,7 +38889,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39378,7 +38901,6 @@ { "fieldPath": "u_lu2", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39391,7 +38913,6 @@ { "fieldPath": "type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39404,7 +38925,6 @@ { "fieldPath": "roles", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39417,7 +38937,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39430,7 +38949,6 @@ { "fieldPath": "default_assignee", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39443,7 +38961,6 @@ { "fieldPath": "email", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39456,7 +38973,6 @@ { "fieldPath": "manager", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39469,7 +38985,6 @@ { "fieldPath": "name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39482,7 +38997,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39495,7 +39009,6 @@ { "fieldPath": "include_members", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39576,7 +39089,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39589,7 +39101,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39602,7 +39113,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39615,7 +39125,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39628,7 +39137,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39641,7 +39149,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39654,7 +39161,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39667,7 +39173,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39680,7 +39185,6 @@ { "fieldPath": "related_incidents", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39693,7 +39197,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39706,7 +39209,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39719,7 +39221,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39732,7 +39233,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39745,7 +39245,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39758,7 +39257,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39771,7 +39269,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39784,7 +39281,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39797,7 +39293,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39810,7 +39305,6 @@ { "fieldPath": "rfc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39823,7 +39317,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39836,7 +39329,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39849,7 +39341,6 @@ { "fieldPath": "problem_state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39862,7 +39353,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39875,7 +39365,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39888,7 +39377,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39901,7 +39389,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39914,7 +39401,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39927,7 +39413,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39940,7 +39425,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39953,7 +39437,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39966,7 +39449,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39979,7 +39461,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39992,7 +39473,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40005,7 +39485,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40018,7 +39497,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40031,7 +39509,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40044,7 +39521,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40057,7 +39533,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40070,7 +39545,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40083,7 +39557,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40096,7 +39569,6 @@ { "fieldPath": "work_around", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40109,7 +39581,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40122,7 +39593,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40135,7 +39605,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40148,7 +39617,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40161,7 +39629,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40174,7 +39641,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40187,7 +39653,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40200,7 +39665,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40213,7 +39677,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40226,7 +39689,6 @@ { "fieldPath": "known_error", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -40239,7 +39701,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40252,7 +39713,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40265,7 +39725,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40278,7 +39737,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40291,7 +39749,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40304,7 +39761,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40317,7 +39773,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40330,7 +39785,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40343,7 +39797,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40356,7 +39809,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40369,7 +39821,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40382,7 +39833,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40395,7 +39845,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40408,7 +39857,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40421,7 +39869,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40434,7 +39881,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40515,7 +39961,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40528,7 +39973,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40541,7 +39985,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40554,7 +39997,6 @@ { "fieldPath": "severity", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40567,7 +40009,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40580,7 +40021,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40593,7 +40033,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40606,7 +40045,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40619,7 +40057,6 @@ { "fieldPath": "parent_incident", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40632,7 +40069,6 @@ { "fieldPath": "subcategory", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40645,7 +40081,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40658,7 +40093,6 @@ { "fieldPath": "caller_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40671,7 +40105,6 @@ { "fieldPath": "resolved_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40684,7 +40117,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40697,7 +40129,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40710,7 +40141,6 @@ { "fieldPath": "resolved_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40723,7 +40153,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40736,7 +40165,6 @@ { "fieldPath": "business_stc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40749,7 +40177,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40762,7 +40189,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40775,7 +40201,6 @@ { "fieldPath": "category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40788,7 +40213,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40801,7 +40225,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40814,7 +40237,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40827,7 +40249,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40840,7 +40261,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40853,7 +40273,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40866,7 +40285,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40879,7 +40297,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40892,7 +40309,6 @@ { "fieldPath": "child_incidents", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40905,7 +40321,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40918,7 +40333,6 @@ { "fieldPath": "incident_state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40931,7 +40345,6 @@ { "fieldPath": "notify", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40944,7 +40357,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40957,7 +40369,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40970,7 +40381,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40983,7 +40393,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40996,7 +40405,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41009,7 +40417,6 @@ { "fieldPath": "problem_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41022,7 +40429,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41035,7 +40441,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41048,7 +40453,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41061,7 +40465,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41074,7 +40477,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41087,7 +40489,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41100,7 +40501,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41113,7 +40513,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41126,7 +40525,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41139,7 +40537,6 @@ { "fieldPath": "caused_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41152,7 +40549,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41165,7 +40561,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41178,7 +40573,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41191,7 +40585,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -41204,7 +40597,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41217,7 +40609,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41230,7 +40621,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41243,7 +40633,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41256,7 +40645,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41269,7 +40657,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41282,7 +40669,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41295,7 +40681,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41308,7 +40693,6 @@ { "fieldPath": "reopen_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41321,7 +40705,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41334,7 +40717,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -41347,7 +40729,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41360,7 +40741,6 @@ { "fieldPath": "calendar_stc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41373,7 +40753,6 @@ { "fieldPath": "rfc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41386,7 +40765,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41399,7 +40777,6 @@ { "fieldPath": "close_code", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41412,7 +40789,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41425,7 +40801,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41438,7 +40813,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41451,7 +40825,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -41464,7 +40837,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41477,7 +40849,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41490,7 +40861,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41503,7 +40873,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41516,7 +40885,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41529,7 +40897,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41610,7 +40977,6 @@ { "fieldPath": "first_discovered", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41623,7 +40989,6 @@ { "fieldPath": "operational_status", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41636,7 +41001,6 @@ { "fieldPath": "last_discovered", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41649,7 +41013,6 @@ { "fieldPath": "cost_cc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41662,7 +41025,6 @@ { "fieldPath": "checked_in", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41675,7 +41037,6 @@ { "fieldPath": "attributes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41688,7 +41049,6 @@ { "fieldPath": "serial_number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41701,7 +41061,6 @@ { "fieldPath": "vendor", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41714,7 +41073,6 @@ { "fieldPath": "ip_address", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41727,7 +41085,6 @@ { "fieldPath": "support_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41740,7 +41097,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41753,7 +41109,6 @@ { "fieldPath": "asset", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41766,7 +41121,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41779,7 +41133,6 @@ { "fieldPath": "supported_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41792,7 +41145,6 @@ { "fieldPath": "invoice_number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41805,7 +41157,6 @@ { "fieldPath": "managed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41818,7 +41169,6 @@ { "fieldPath": "fault_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41831,7 +41181,6 @@ { "fieldPath": "due_in", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41844,7 +41193,6 @@ { "fieldPath": "cost", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41857,7 +41205,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41870,7 +41217,6 @@ { "fieldPath": "justification", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41883,7 +41229,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41896,7 +41241,6 @@ { "fieldPath": "assigned", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41909,7 +41253,6 @@ { "fieldPath": "model_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41922,7 +41265,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41935,7 +41277,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41948,7 +41289,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41961,7 +41301,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41974,7 +41313,6 @@ { "fieldPath": "lease_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41987,7 +41325,6 @@ { "fieldPath": "monitor", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -42000,7 +41337,6 @@ { "fieldPath": "cost_center", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42013,7 +41349,6 @@ { "fieldPath": "subcategory", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42026,7 +41361,6 @@ { "fieldPath": "delivery_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42039,7 +41373,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42052,7 +41385,6 @@ { "fieldPath": "can_print", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -42065,7 +41397,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42078,7 +41409,6 @@ { "fieldPath": "model_number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42091,7 +41421,6 @@ { "fieldPath": "name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42104,7 +41433,6 @@ { "fieldPath": "start_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42117,7 +41445,6 @@ { "fieldPath": "discovery_source", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42130,7 +41457,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42143,7 +41469,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42156,7 +41481,6 @@ { "fieldPath": "category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42169,7 +41493,6 @@ { "fieldPath": "schedule", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42182,7 +41505,6 @@ { "fieldPath": "fqdn", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42195,7 +41517,6 @@ { "fieldPath": "warranty_expiration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -42208,7 +41529,6 @@ { "fieldPath": "owned_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42221,7 +41541,6 @@ { "fieldPath": "asset_tag", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42234,7 +41553,6 @@ { "fieldPath": "manufacturer", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42247,7 +41565,6 @@ { "fieldPath": "purchase_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -42260,7 +41577,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42273,7 +41589,6 @@ { "fieldPath": "department", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42286,7 +41601,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42299,7 +41613,6 @@ { "fieldPath": "checked_out", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42312,7 +41625,6 @@ { "fieldPath": "unverified", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -42325,7 +41637,6 @@ { "fieldPath": "skip_sync", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -42338,7 +41649,6 @@ { "fieldPath": "po_number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42351,7 +41661,6 @@ { "fieldPath": "order_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42364,7 +41673,6 @@ { "fieldPath": "gl_account", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42377,7 +41685,6 @@ { "fieldPath": "maintenance_schedule", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42390,7 +41697,6 @@ { "fieldPath": "install_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42403,7 +41709,6 @@ { "fieldPath": "dns_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42416,7 +41721,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42429,7 +41733,6 @@ { "fieldPath": "mac_address", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42442,7 +41745,6 @@ { "fieldPath": "change_control", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42455,7 +41757,6 @@ { "fieldPath": "install_status", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -42468,7 +41769,6 @@ { "fieldPath": "due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42481,7 +41781,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} diff --git a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden_deleted_stateful.json b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden_deleted_stateful.json index 96df70dd5b02a..08d5127b18041 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden_deleted_stateful.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden_deleted_stateful.json @@ -243,8 +243,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "entityType": "tag", + "entityUrn": "urn:li:tag:GROUPFIELD", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -259,7 +259,7 @@ }, { "entityType": "tag", - "entityUrn": "urn:li:tag:GROUPFIELD", + "entityUrn": "urn:li:tag:ATTRIBUTE", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -273,8 +273,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "entityType": "tag", + "entityUrn": "urn:li:tag:SUM", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -288,8 +288,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -304,7 +304,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -319,7 +319,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -333,8 +333,8 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -348,8 +348,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "entityType": "tag", + "entityUrn": "urn:li:tag:DATASOURCEFIELD", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -363,8 +363,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -378,8 +378,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -393,8 +393,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -408,8 +408,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -438,8 +438,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -454,7 +454,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -468,8 +468,8 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:BINFIELD", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -484,7 +484,7 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -498,8 +498,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -513,8 +513,8 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:TagSheet3", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -528,8 +528,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "entityType": "tag", + "entityUrn": "urn:li:tag:SETFIELD", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -543,8 +543,8 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -558,8 +558,8 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -574,7 +574,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -589,7 +589,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -603,8 +603,8 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:COLUMNFIELD", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -619,7 +619,7 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -634,7 +634,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -648,8 +648,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "entityType": "tag", + "entityUrn": "urn:li:tag:HIERARCHYFIELD", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -664,7 +664,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -678,8 +678,8 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:CALCULATEDFIELD", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -693,8 +693,8 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -708,8 +708,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -724,7 +724,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -739,7 +739,7 @@ }, { "entityType": "tag", - "entityUrn": "urn:li:tag:SUM", + "entityUrn": "urn:li:tag:YEAR", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -753,8 +753,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -768,8 +768,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -783,8 +783,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -798,8 +798,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "entityType": "tag", + "entityUrn": "urn:li:tag:CALCULATEDFIELD", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -813,8 +813,8 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:HIERARCHYFIELD", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -829,7 +829,7 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -843,8 +843,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -858,8 +858,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -873,8 +873,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -888,8 +888,8 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:DATASOURCEFIELD", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -903,8 +903,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -918,8 +918,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "entityType": "tag", + "entityUrn": "urn:li:tag:TagSheet3", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -934,7 +934,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -949,7 +949,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -963,8 +963,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -978,8 +978,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -994,7 +994,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1009,7 +1009,7 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1023,8 +1023,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "entityType": "tag", + "entityUrn": "urn:li:tag:BINFIELD", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1039,7 +1039,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1053,8 +1053,8 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1068,8 +1068,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "entityType": "tag", + "entityUrn": "urn:li:tag:COUNT", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1083,8 +1083,8 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:DIMENSION", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1098,8 +1098,8 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:YEAR", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1114,7 +1114,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1128,8 +1128,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1143,8 +1143,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "entityType": "tag", + "entityUrn": "urn:li:tag:COLUMNFIELD", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1159,7 +1159,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1173,8 +1173,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1189,7 +1189,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1203,8 +1203,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1219,7 +1219,7 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1233,8 +1233,8 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1248,8 +1248,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1263,8 +1263,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "entityType": "tag", + "entityUrn": "urn:li:tag:DIMENSION", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1278,8 +1278,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1293,8 +1293,8 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:COUNT", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1309,7 +1309,7 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1323,8 +1323,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1338,8 +1338,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1353,8 +1353,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1384,7 +1384,7 @@ }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1398,8 +1398,8 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:SETFIELD", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1413,8 +1413,8 @@ } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:ATTRIBUTE", + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", "changeType": "UPSERT", "aspectName": "status", "aspect": { diff --git a/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json index 665541a140cc9..e6782c1c9f5f3 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json @@ -1308,9 +1308,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign List", "title": "Campaign List", "description": "", @@ -1869,9 +1867,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Summary", "title": "Summary", "description": "", @@ -2482,9 +2478,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile - Sent by Campaign", "title": "Mobile - Sent by Campaign", "description": "", @@ -2991,9 +2985,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet1", "title": "Sheet 1", "description": "", @@ -3136,9 +3128,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet2", "title": "Sheet 2", "description": "", @@ -3449,9 +3439,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet3", "title": "Sheet 3", "description": "", @@ -3646,9 +3634,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Requests", "title": "Opened Requests", "description": "", @@ -4147,9 +4133,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top 10 Items by Requests and YoY Change", "title": "Top 10 Items by Requests and YoY Change", "description": "", @@ -4616,9 +4600,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Problems", "title": "Opened Problems", "description": "", @@ -5117,9 +5099,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue", "title": "Overdue", "description": "", @@ -5499,9 +5479,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High and Critical Priority Problems", "title": "High and Critical Priority Problems", "description": "", @@ -5861,9 +5839,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total Incidents by Category and YoY Change", "title": "Total Incidents by Category and YoY Change", "description": "", @@ -6246,9 +6222,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known Errors", "title": "Known Errors", "description": "", @@ -6686,9 +6660,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Requests", "title": "Overdue Requests", "description": "", @@ -7100,9 +7072,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG Time to Solve an Incident", "title": "AVG Time to Solve an Incident", "description": "", @@ -7430,9 +7400,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made SLA?", "title": "Made SLA?", "description": "", @@ -7792,9 +7760,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-IncidentBreakdownbyPriority", "title": "Tooltip - Incident Breakdown by Priority", "description": "", @@ -8044,9 +8010,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Problems", "title": "Overdue Problems", "description": "", @@ -8484,9 +8448,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-ProblemBreakdownbyPriority", "title": "Tooltip - Problem Breakdown by Priority", "description": "", @@ -8820,9 +8782,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-RequestBreakdownbyPriority", "title": "Tooltip - Request Breakdown by Priority", "description": "", @@ -9182,9 +9142,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age of Active Problems", "title": "Age of Active Problems", "description": "", @@ -9570,9 +9528,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Incidents", "title": "Opened Incidents", "description": "", @@ -10010,9 +9966,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/Workbookpublishedds/Sheet1", "title": "published sheet ds", "description": "", @@ -10323,9 +10277,7 @@ }, { "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "title": "dvd Rental Dashboard", "description": "", "charts": [ @@ -10425,9 +10377,7 @@ }, { "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "title": "Story 1", "description": "", "charts": [], @@ -10525,9 +10475,7 @@ }, { "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "title": "Executive Dashboard", "description": "", "charts": [ @@ -11006,9 +10954,7 @@ "customProperties": { "hasExtracts": "True", "extractLastRefreshTime": "2018-02-09T00:05:25Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-02-09T00:05:25Z", - "type": "EmbeddedDatasource" + "extractLastUpdateTime": "2018-02-09T00:05:25Z" }, "name": "Marketo", "tags": [] @@ -12943,13 +12889,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, + "customProperties": {}, "name": "Customer Payment Query", "tags": [] } @@ -13279,13 +13219,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, + "customProperties": {}, "name": "actor+ (dvdrental)", "tags": [] } @@ -14189,13 +14123,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, + "customProperties": {}, "name": "Superstore Datasource", "tags": [] } @@ -15148,9 +15076,7 @@ "customProperties": { "hasExtracts": "True", "extractLastRefreshTime": "2018-01-18T19:35:39Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-01-18T19:35:39Z", - "type": "EmbeddedDatasource" + "extractLastUpdateTime": "2018-01-18T19:35:39Z" }, "name": "Requests", "tags": [] @@ -22011,9 +21937,7 @@ "customProperties": { "hasExtracts": "True", "extractLastRefreshTime": "2018-01-18T20:21:33Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-01-18T20:21:33Z", - "type": "EmbeddedDatasource" + "extractLastUpdateTime": "2018-01-18T20:21:33Z" }, "name": "Problems", "tags": [] @@ -26122,9 +26046,7 @@ "customProperties": { "hasExtracts": "True", "extractLastRefreshTime": "2018-01-18T20:13:08Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-01-18T20:13:08Z", - "type": "EmbeddedDatasource" + "extractLastUpdateTime": "2018-01-18T20:13:08Z" }, "name": "Incidents", "tags": [] @@ -31464,13 +31386,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, + "customProperties": {}, "name": "test publish datasource", "tags": [] } @@ -31858,13 +31774,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "PublishedDatasource" - }, + "customProperties": {}, "name": "test publish datasource", "description": "description for test publish datasource", "tags": [] @@ -32324,13 +32234,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "PublishedDatasource" - }, + "customProperties": {}, "name": "Superstore Datasource", "description": "Description for Superstore dataset", "tags": [] @@ -33712,7 +33616,6 @@ { "fieldPath": "Test_Variant", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33725,7 +33628,6 @@ { "fieldPath": "Mailing_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33738,7 +33640,6 @@ { "fieldPath": "Campaign_Run_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33751,7 +33652,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33764,7 +33664,6 @@ { "fieldPath": "Activity_Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -33777,7 +33676,6 @@ { "fieldPath": "Choice_Number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33790,7 +33688,6 @@ { "fieldPath": "Step_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33803,7 +33700,6 @@ { "fieldPath": "Campaign_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33816,7 +33712,6 @@ { "fieldPath": "Lead_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33829,7 +33724,6 @@ { "fieldPath": "Has_Predictive", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -33910,7 +33804,6 @@ { "fieldPath": "Campaign_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33923,7 +33816,6 @@ { "fieldPath": "Campaign_Run_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33936,7 +33828,6 @@ { "fieldPath": "Link", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -33949,7 +33840,6 @@ { "fieldPath": "Test_Variant", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33962,7 +33852,6 @@ { "fieldPath": "Platform", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -33975,7 +33864,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33988,7 +33876,6 @@ { "fieldPath": "Activity_Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34001,7 +33888,6 @@ { "fieldPath": "Choice_Number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34014,7 +33900,6 @@ { "fieldPath": "Mailing_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34027,7 +33912,6 @@ { "fieldPath": "Step_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34040,7 +33924,6 @@ { "fieldPath": "Lead_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34053,7 +33936,6 @@ { "fieldPath": "Link_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34066,7 +33948,6 @@ { "fieldPath": "Is_Predictive", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34079,7 +33960,6 @@ { "fieldPath": "Device", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34092,7 +33972,6 @@ { "fieldPath": "Is_Mobile_Device", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34105,7 +33984,6 @@ { "fieldPath": "User_Agent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34186,7 +34064,6 @@ { "fieldPath": "Platform", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34199,7 +34076,6 @@ { "fieldPath": "Device", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34212,7 +34088,6 @@ { "fieldPath": "Choice_Number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34225,7 +34100,6 @@ { "fieldPath": "Lead_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34238,7 +34112,6 @@ { "fieldPath": "Activity_Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34251,7 +34124,6 @@ { "fieldPath": "Test_Variant", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34264,7 +34136,6 @@ { "fieldPath": "Is_Mobile_Device", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34277,7 +34148,6 @@ { "fieldPath": "Has_Predictive", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34290,7 +34160,6 @@ { "fieldPath": "Step_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34303,7 +34172,6 @@ { "fieldPath": "User_Agent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34316,7 +34184,6 @@ { "fieldPath": "Mailing_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34329,7 +34196,6 @@ { "fieldPath": "Campaign_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34342,7 +34208,6 @@ { "fieldPath": "Campaign_Run_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34355,7 +34220,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34436,7 +34300,6 @@ { "fieldPath": "Test_Variant", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34449,7 +34312,6 @@ { "fieldPath": "Campaign_Run_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34462,7 +34324,6 @@ { "fieldPath": "Activity_Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34475,7 +34336,6 @@ { "fieldPath": "Mailing_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34488,7 +34348,6 @@ { "fieldPath": "Has_Predictive", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34501,7 +34360,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34514,7 +34372,6 @@ { "fieldPath": "Campaign_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34527,7 +34384,6 @@ { "fieldPath": "Step_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34540,7 +34396,6 @@ { "fieldPath": "Lead_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34553,7 +34408,6 @@ { "fieldPath": "Choice_Number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34634,7 +34488,6 @@ { "fieldPath": "programName", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34647,7 +34500,6 @@ { "fieldPath": "name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34660,7 +34512,6 @@ { "fieldPath": "programId", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34673,7 +34524,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34686,7 +34536,6 @@ { "fieldPath": "createdAt", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34699,7 +34548,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34712,7 +34560,6 @@ { "fieldPath": "workspaceName", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34725,7 +34572,6 @@ { "fieldPath": "updatedAt", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34738,7 +34584,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34819,7 +34664,6 @@ { "fieldPath": "postal_code", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34832,7 +34676,6 @@ { "fieldPath": "last_update", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34845,7 +34688,6 @@ { "fieldPath": "phone", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34858,7 +34700,6 @@ { "fieldPath": "address2", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34871,7 +34712,6 @@ { "fieldPath": "address", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34884,7 +34724,6 @@ { "fieldPath": "city_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34897,7 +34736,6 @@ { "fieldPath": "district", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34910,7 +34748,6 @@ { "fieldPath": "address_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34991,7 +34828,6 @@ { "fieldPath": "last_update", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35004,7 +34840,6 @@ { "fieldPath": "last_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35017,7 +34852,6 @@ { "fieldPath": "first_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35030,7 +34864,6 @@ { "fieldPath": "actor_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35111,7 +34944,6 @@ { "fieldPath": "Person", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35124,7 +34956,6 @@ { "fieldPath": "Region", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35202,7 +35033,6 @@ { "fieldPath": "Returned", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35215,7 +35045,6 @@ { "fieldPath": "Order ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35293,7 +35122,6 @@ { "fieldPath": "Product ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35306,7 +35134,6 @@ { "fieldPath": "Category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35319,7 +35146,6 @@ { "fieldPath": "Postal Code", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35332,7 +35158,6 @@ { "fieldPath": "City", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35345,7 +35170,6 @@ { "fieldPath": "Quantity", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35358,7 +35182,6 @@ { "fieldPath": "State", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35371,7 +35194,6 @@ { "fieldPath": "Order Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -35384,7 +35206,6 @@ { "fieldPath": "Customer Name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35397,7 +35218,6 @@ { "fieldPath": "Country/Region", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35410,7 +35230,6 @@ { "fieldPath": "Sales", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35423,7 +35242,6 @@ { "fieldPath": "Segment", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35436,7 +35254,6 @@ { "fieldPath": "Sub-Category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35449,7 +35266,6 @@ { "fieldPath": "Profit", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35462,7 +35278,6 @@ { "fieldPath": "Product Name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35475,7 +35290,6 @@ { "fieldPath": "Customer ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35488,7 +35302,6 @@ { "fieldPath": "Order ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35501,7 +35314,6 @@ { "fieldPath": "Row ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35514,7 +35326,6 @@ { "fieldPath": "Discount", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35527,7 +35338,6 @@ { "fieldPath": "Ship Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -35540,7 +35350,6 @@ { "fieldPath": "Ship Mode", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35553,7 +35362,6 @@ { "fieldPath": "Region", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35633,7 +35441,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35646,7 +35453,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35659,7 +35465,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35672,7 +35477,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35685,7 +35489,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35698,7 +35501,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35711,7 +35513,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35724,7 +35525,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35737,7 +35537,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35750,7 +35549,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35763,7 +35561,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35776,7 +35573,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -35789,7 +35585,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35802,7 +35597,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -35815,7 +35609,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35828,7 +35621,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35841,7 +35633,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35854,7 +35645,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35867,7 +35657,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35880,7 +35669,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35893,7 +35681,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35906,7 +35693,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35919,7 +35705,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35932,7 +35717,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35945,7 +35729,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35958,7 +35741,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35971,7 +35753,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35984,7 +35765,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35997,7 +35777,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36010,7 +35789,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36023,7 +35801,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36036,7 +35813,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36049,7 +35825,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36062,7 +35837,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36075,7 +35849,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36088,7 +35861,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36101,7 +35873,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36114,7 +35885,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36127,7 +35897,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36140,7 +35909,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36153,7 +35921,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36166,7 +35933,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36179,7 +35945,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36192,7 +35957,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36205,7 +35969,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36218,7 +35981,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -36231,7 +35993,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36244,7 +36005,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36257,7 +36017,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36270,7 +36029,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36283,7 +36041,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36296,7 +36053,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36309,7 +36065,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36322,7 +36077,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36335,7 +36089,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36348,7 +36101,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36361,7 +36113,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36374,7 +36125,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36387,7 +36137,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36400,7 +36149,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36413,7 +36161,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36426,7 +36173,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36507,7 +36253,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36520,7 +36265,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36533,7 +36277,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36546,7 +36289,6 @@ { "fieldPath": "requested_for", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36559,7 +36301,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36572,7 +36313,6 @@ { "fieldPath": "price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36585,7 +36325,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36598,7 +36337,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36611,7 +36349,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36624,7 +36361,6 @@ { "fieldPath": "delivery_address", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36637,7 +36373,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36650,7 +36385,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36663,7 +36397,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36676,7 +36409,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36689,7 +36421,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36702,7 +36433,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -36715,7 +36445,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36728,7 +36457,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36741,7 +36469,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36754,7 +36481,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36767,7 +36493,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36780,7 +36505,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36793,7 +36517,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36806,7 +36529,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36819,7 +36541,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36832,7 +36553,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36845,7 +36565,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36858,7 +36577,6 @@ { "fieldPath": "requested_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -36871,7 +36589,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36884,7 +36601,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36897,7 +36613,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36910,7 +36625,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36923,7 +36637,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36936,7 +36649,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36949,7 +36661,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36962,7 +36673,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36975,7 +36685,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36988,7 +36697,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37001,7 +36709,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37014,7 +36721,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37027,7 +36733,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37040,7 +36745,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37053,7 +36757,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37066,7 +36769,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37079,7 +36781,6 @@ { "fieldPath": "request_state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37092,7 +36793,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37105,7 +36805,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37118,7 +36817,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37131,7 +36829,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37144,7 +36841,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37157,7 +36853,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37170,7 +36865,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37183,7 +36877,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37196,7 +36889,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37209,7 +36901,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37222,7 +36913,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37235,7 +36925,6 @@ { "fieldPath": "stage", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37248,7 +36937,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37261,7 +36949,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37274,7 +36961,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37287,7 +36973,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37300,7 +36985,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37313,7 +36997,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37326,7 +37009,6 @@ { "fieldPath": "special_instructions", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37339,7 +37021,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37352,7 +37033,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37365,7 +37045,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37378,7 +37057,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37391,7 +37069,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37404,7 +37081,6 @@ { "fieldPath": "calendar_stc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37485,7 +37161,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37498,7 +37173,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37511,7 +37185,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37524,7 +37197,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37537,7 +37209,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37550,7 +37221,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37563,7 +37233,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37576,7 +37245,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37589,7 +37257,6 @@ { "fieldPath": "configuration_item", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37602,7 +37269,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37615,7 +37281,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37628,7 +37293,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37641,7 +37305,6 @@ { "fieldPath": "recurring_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37654,7 +37317,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37667,7 +37329,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37680,7 +37341,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37693,7 +37353,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37706,7 +37365,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37719,7 +37377,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37732,7 +37389,6 @@ { "fieldPath": "sc_catalog", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37745,7 +37401,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37758,7 +37413,6 @@ { "fieldPath": "stage", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37771,7 +37425,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37784,7 +37437,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37797,7 +37449,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37810,7 +37461,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37823,7 +37473,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37836,7 +37485,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37849,7 +37497,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37862,7 +37509,6 @@ { "fieldPath": "price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37875,7 +37521,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37888,7 +37533,6 @@ { "fieldPath": "estimated_delivery", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37901,7 +37545,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37914,7 +37557,6 @@ { "fieldPath": "context", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37927,7 +37569,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37940,7 +37581,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37953,7 +37593,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37966,7 +37605,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37979,7 +37617,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37992,7 +37629,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38005,7 +37641,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38018,7 +37653,6 @@ { "fieldPath": "billable", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38031,7 +37665,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38044,7 +37677,6 @@ { "fieldPath": "recurring_frequency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38057,7 +37689,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38070,7 +37701,6 @@ { "fieldPath": "cat_item", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38083,7 +37713,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38096,7 +37725,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38109,7 +37737,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38122,7 +37749,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38135,7 +37761,6 @@ { "fieldPath": "order_guide", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38148,7 +37773,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38161,7 +37785,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38174,7 +37797,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38187,7 +37809,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38200,7 +37821,6 @@ { "fieldPath": "backordered", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38213,7 +37833,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38226,7 +37845,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38239,7 +37857,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38252,7 +37869,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38265,7 +37881,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38278,7 +37893,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38291,7 +37905,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38304,7 +37917,6 @@ { "fieldPath": "request", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38317,7 +37929,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38330,7 +37941,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38343,7 +37953,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38356,7 +37965,6 @@ { "fieldPath": "quantity", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38369,7 +37977,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38382,7 +37989,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38395,7 +38001,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38408,7 +38013,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38421,7 +38025,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38434,7 +38037,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38447,7 +38049,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38460,7 +38061,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38541,7 +38141,6 @@ { "fieldPath": "sc_catalogs", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38554,7 +38153,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38567,7 +38165,6 @@ { "fieldPath": "type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38580,7 +38177,6 @@ { "fieldPath": "mobile_picture_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38593,7 +38189,6 @@ { "fieldPath": "workflow", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38606,7 +38201,6 @@ { "fieldPath": "sys_customer_update", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38619,7 +38213,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38632,7 +38225,6 @@ { "fieldPath": "visible_standalone", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38645,7 +38237,6 @@ { "fieldPath": "no_quantity", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38658,7 +38249,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38671,7 +38261,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38684,7 +38273,6 @@ { "fieldPath": "sys_scope", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38697,7 +38285,6 @@ { "fieldPath": "template", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38710,7 +38297,6 @@ { "fieldPath": "no_proceed_checkout", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38723,7 +38309,6 @@ { "fieldPath": "billable", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38736,7 +38321,6 @@ { "fieldPath": "name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38749,7 +38333,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38762,7 +38345,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38775,7 +38357,6 @@ { "fieldPath": "meta", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38788,7 +38369,6 @@ { "fieldPath": "ordered_item_link", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38801,7 +38381,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38814,7 +38393,6 @@ { "fieldPath": "sc_ic_version", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38827,7 +38405,6 @@ { "fieldPath": "image", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38840,7 +38417,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38853,7 +38429,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38866,7 +38441,6 @@ { "fieldPath": "sys_policy", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38879,7 +38453,6 @@ { "fieldPath": "roles", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38892,7 +38465,6 @@ { "fieldPath": "picture", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38905,7 +38477,6 @@ { "fieldPath": "list_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38918,7 +38489,6 @@ { "fieldPath": "no_order_now", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38931,7 +38501,6 @@ { "fieldPath": "vendor", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38944,7 +38513,6 @@ { "fieldPath": "sc_ic_item_staging", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38957,7 +38525,6 @@ { "fieldPath": "no_order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38970,7 +38537,6 @@ { "fieldPath": "entitlement_script", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38983,7 +38549,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38996,7 +38561,6 @@ { "fieldPath": "icon", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39009,7 +38573,6 @@ { "fieldPath": "ignore_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39022,7 +38585,6 @@ { "fieldPath": "start_closed", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39035,7 +38597,6 @@ { "fieldPath": "sys_package", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39048,7 +38609,6 @@ { "fieldPath": "group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39061,7 +38621,6 @@ { "fieldPath": "sys_replace_on_upgrade", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39074,7 +38633,6 @@ { "fieldPath": "cost", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39087,7 +38645,6 @@ { "fieldPath": "use_sc_layout", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39100,7 +38657,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39113,7 +38669,6 @@ { "fieldPath": "availability", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39126,7 +38681,6 @@ { "fieldPath": "model", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39139,7 +38693,6 @@ { "fieldPath": "custom_cart", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39152,7 +38705,6 @@ { "fieldPath": "mobile_picture", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39165,7 +38717,6 @@ { "fieldPath": "category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39178,7 +38729,6 @@ { "fieldPath": "no_cart", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39191,7 +38741,6 @@ { "fieldPath": "mobile_hide_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39204,7 +38753,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39217,7 +38765,6 @@ { "fieldPath": "price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39230,7 +38777,6 @@ { "fieldPath": "delivery_time", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39243,7 +38789,6 @@ { "fieldPath": "no_search", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39256,7 +38801,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39269,7 +38813,6 @@ { "fieldPath": "recurring_frequency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39282,7 +38825,6 @@ { "fieldPath": "recurring_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39295,7 +38837,6 @@ { "fieldPath": "delivery_plan_script", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39308,7 +38849,6 @@ { "fieldPath": "visible_bundle", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39321,7 +38861,6 @@ { "fieldPath": "sys_update_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39334,7 +38873,6 @@ { "fieldPath": "sys_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39347,7 +38885,6 @@ { "fieldPath": "visible_guide", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39360,7 +38897,6 @@ { "fieldPath": "preview", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39373,7 +38909,6 @@ { "fieldPath": "omit_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39454,7 +38989,6 @@ { "fieldPath": "u_u", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39467,7 +39001,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39480,7 +39013,6 @@ { "fieldPath": "source", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39493,7 +39025,6 @@ { "fieldPath": "exclude_manager", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39506,7 +39037,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39519,7 +39049,6 @@ { "fieldPath": "cost_center", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39532,7 +39061,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39545,7 +39073,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39558,7 +39085,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39571,7 +39097,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39584,7 +39109,6 @@ { "fieldPath": "u_lucha", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39597,7 +39121,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39610,7 +39133,6 @@ { "fieldPath": "u_lu2", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39623,7 +39145,6 @@ { "fieldPath": "type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39636,7 +39157,6 @@ { "fieldPath": "roles", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39649,7 +39169,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39662,7 +39181,6 @@ { "fieldPath": "default_assignee", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39675,7 +39193,6 @@ { "fieldPath": "email", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39688,7 +39205,6 @@ { "fieldPath": "manager", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39701,7 +39217,6 @@ { "fieldPath": "name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39714,7 +39229,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39727,7 +39241,6 @@ { "fieldPath": "include_members", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39808,7 +39321,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39821,7 +39333,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39834,7 +39345,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39847,7 +39357,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39860,7 +39369,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39873,7 +39381,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39886,7 +39393,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39899,7 +39405,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39912,7 +39417,6 @@ { "fieldPath": "related_incidents", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39925,7 +39429,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39938,7 +39441,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39951,7 +39453,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39964,7 +39465,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39977,7 +39477,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39990,7 +39489,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40003,7 +39501,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40016,7 +39513,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40029,7 +39525,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40042,7 +39537,6 @@ { "fieldPath": "rfc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40055,7 +39549,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40068,7 +39561,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40081,7 +39573,6 @@ { "fieldPath": "problem_state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40094,7 +39585,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40107,7 +39597,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40120,7 +39609,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -40133,7 +39621,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40146,7 +39633,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40159,7 +39645,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40172,7 +39657,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40185,7 +39669,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40198,7 +39681,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40211,7 +39693,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -40224,7 +39705,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40237,7 +39717,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40250,7 +39729,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40263,7 +39741,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40276,7 +39753,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40289,7 +39765,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40302,7 +39777,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40315,7 +39789,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40328,7 +39801,6 @@ { "fieldPath": "work_around", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40341,7 +39813,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40354,7 +39825,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40367,7 +39837,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40380,7 +39849,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40393,7 +39861,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40406,7 +39873,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40419,7 +39885,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40432,7 +39897,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40445,7 +39909,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40458,7 +39921,6 @@ { "fieldPath": "known_error", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -40471,7 +39933,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40484,7 +39945,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40497,7 +39957,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40510,7 +39969,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40523,7 +39981,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40536,7 +39993,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40549,7 +40005,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40562,7 +40017,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40575,7 +40029,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40588,7 +40041,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40601,7 +40053,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40614,7 +40065,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40627,7 +40077,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40640,7 +40089,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40653,7 +40101,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40666,7 +40113,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40747,7 +40193,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40760,7 +40205,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40773,7 +40217,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40786,7 +40229,6 @@ { "fieldPath": "severity", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40799,7 +40241,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40812,7 +40253,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40825,7 +40265,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40838,7 +40277,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40851,7 +40289,6 @@ { "fieldPath": "parent_incident", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40864,7 +40301,6 @@ { "fieldPath": "subcategory", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40877,7 +40313,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40890,7 +40325,6 @@ { "fieldPath": "caller_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40903,7 +40337,6 @@ { "fieldPath": "resolved_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40916,7 +40349,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40929,7 +40361,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40942,7 +40373,6 @@ { "fieldPath": "resolved_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40955,7 +40385,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40968,7 +40397,6 @@ { "fieldPath": "business_stc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40981,7 +40409,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40994,7 +40421,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41007,7 +40433,6 @@ { "fieldPath": "category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41020,7 +40445,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41033,7 +40457,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41046,7 +40469,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41059,7 +40481,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41072,7 +40493,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41085,7 +40505,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41098,7 +40517,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41111,7 +40529,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41124,7 +40541,6 @@ { "fieldPath": "child_incidents", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41137,7 +40553,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41150,7 +40565,6 @@ { "fieldPath": "incident_state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41163,7 +40577,6 @@ { "fieldPath": "notify", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41176,7 +40589,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41189,7 +40601,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41202,7 +40613,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41215,7 +40625,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41228,7 +40637,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41241,7 +40649,6 @@ { "fieldPath": "problem_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41254,7 +40661,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41267,7 +40673,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41280,7 +40685,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41293,7 +40697,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41306,7 +40709,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41319,7 +40721,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41332,7 +40733,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41345,7 +40745,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41358,7 +40757,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41371,7 +40769,6 @@ { "fieldPath": "caused_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41384,7 +40781,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41397,7 +40793,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41410,7 +40805,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41423,7 +40817,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -41436,7 +40829,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41449,7 +40841,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41462,7 +40853,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41475,7 +40865,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41488,7 +40877,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41501,7 +40889,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41514,7 +40901,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41527,7 +40913,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41540,7 +40925,6 @@ { "fieldPath": "reopen_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41553,7 +40937,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41566,7 +40949,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -41579,7 +40961,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41592,7 +40973,6 @@ { "fieldPath": "calendar_stc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41605,7 +40985,6 @@ { "fieldPath": "rfc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41618,7 +40997,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41631,7 +41009,6 @@ { "fieldPath": "close_code", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41644,7 +41021,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41657,7 +41033,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41670,7 +41045,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41683,7 +41057,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -41696,7 +41069,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41709,7 +41081,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41722,7 +41093,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41735,7 +41105,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41748,7 +41117,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41761,7 +41129,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41842,7 +41209,6 @@ { "fieldPath": "first_discovered", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41855,7 +41221,6 @@ { "fieldPath": "operational_status", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41868,7 +41233,6 @@ { "fieldPath": "last_discovered", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41881,7 +41245,6 @@ { "fieldPath": "cost_cc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41894,7 +41257,6 @@ { "fieldPath": "checked_in", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41907,7 +41269,6 @@ { "fieldPath": "attributes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41920,7 +41281,6 @@ { "fieldPath": "serial_number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41933,7 +41293,6 @@ { "fieldPath": "vendor", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41946,7 +41305,6 @@ { "fieldPath": "ip_address", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41959,7 +41317,6 @@ { "fieldPath": "support_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41972,7 +41329,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41985,7 +41341,6 @@ { "fieldPath": "asset", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41998,7 +41353,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42011,7 +41365,6 @@ { "fieldPath": "supported_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42024,7 +41377,6 @@ { "fieldPath": "invoice_number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42037,7 +41389,6 @@ { "fieldPath": "managed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42050,7 +41401,6 @@ { "fieldPath": "fault_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -42063,7 +41413,6 @@ { "fieldPath": "due_in", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42076,7 +41425,6 @@ { "fieldPath": "cost", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42089,7 +41437,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42102,7 +41449,6 @@ { "fieldPath": "justification", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42115,7 +41461,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42128,7 +41473,6 @@ { "fieldPath": "assigned", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42141,7 +41485,6 @@ { "fieldPath": "model_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42154,7 +41497,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42167,7 +41509,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42180,7 +41521,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42193,7 +41533,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42206,7 +41545,6 @@ { "fieldPath": "lease_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42219,7 +41557,6 @@ { "fieldPath": "monitor", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -42232,7 +41569,6 @@ { "fieldPath": "cost_center", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42245,7 +41581,6 @@ { "fieldPath": "subcategory", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42258,7 +41593,6 @@ { "fieldPath": "delivery_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42271,7 +41605,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42284,7 +41617,6 @@ { "fieldPath": "can_print", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -42297,7 +41629,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42310,7 +41641,6 @@ { "fieldPath": "model_number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42323,7 +41653,6 @@ { "fieldPath": "name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42336,7 +41665,6 @@ { "fieldPath": "start_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42349,7 +41677,6 @@ { "fieldPath": "discovery_source", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42362,7 +41689,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42375,7 +41701,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42388,7 +41713,6 @@ { "fieldPath": "category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42401,7 +41725,6 @@ { "fieldPath": "schedule", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42414,7 +41737,6 @@ { "fieldPath": "fqdn", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42427,7 +41749,6 @@ { "fieldPath": "warranty_expiration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -42440,7 +41761,6 @@ { "fieldPath": "owned_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42453,7 +41773,6 @@ { "fieldPath": "asset_tag", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42466,7 +41785,6 @@ { "fieldPath": "manufacturer", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42479,7 +41797,6 @@ { "fieldPath": "purchase_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -42492,7 +41809,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42505,7 +41821,6 @@ { "fieldPath": "department", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42518,7 +41833,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42531,7 +41845,6 @@ { "fieldPath": "checked_out", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42544,7 +41857,6 @@ { "fieldPath": "unverified", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -42557,7 +41869,6 @@ { "fieldPath": "skip_sync", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -42570,7 +41881,6 @@ { "fieldPath": "po_number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42583,7 +41893,6 @@ { "fieldPath": "order_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42596,7 +41905,6 @@ { "fieldPath": "gl_account", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42609,7 +41917,6 @@ { "fieldPath": "maintenance_schedule", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42622,7 +41929,6 @@ { "fieldPath": "install_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42635,7 +41941,6 @@ { "fieldPath": "dns_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42648,7 +41953,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42661,7 +41965,6 @@ { "fieldPath": "mac_address", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42674,7 +41977,6 @@ { "fieldPath": "change_control", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42687,7 +41989,6 @@ { "fieldPath": "install_status", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -42700,7 +42001,6 @@ { "fieldPath": "due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42713,7 +42013,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} diff --git a/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json index 45b1ebca25b8a..ea15a2ac56b58 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json @@ -1076,9 +1076,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign List", "title": "Campaign List", "description": "", @@ -1637,9 +1635,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Summary", "title": "Summary", "description": "", @@ -2250,9 +2246,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile - Sent by Campaign", "title": "Mobile - Sent by Campaign", "description": "", @@ -2759,9 +2753,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet1", "title": "Sheet 1", "description": "", @@ -2904,9 +2896,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet2", "title": "Sheet 2", "description": "", @@ -3217,9 +3207,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet3", "title": "Sheet 3", "description": "", @@ -3414,9 +3402,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Requests", "title": "Opened Requests", "description": "", @@ -3915,9 +3901,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top 10 Items by Requests and YoY Change", "title": "Top 10 Items by Requests and YoY Change", "description": "", @@ -4384,9 +4368,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Problems", "title": "Opened Problems", "description": "", @@ -4885,9 +4867,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue", "title": "Overdue", "description": "", @@ -5267,9 +5247,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High and Critical Priority Problems", "title": "High and Critical Priority Problems", "description": "", @@ -5629,9 +5607,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total Incidents by Category and YoY Change", "title": "Total Incidents by Category and YoY Change", "description": "", @@ -6014,9 +5990,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known Errors", "title": "Known Errors", "description": "", @@ -6454,9 +6428,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Requests", "title": "Overdue Requests", "description": "", @@ -6868,9 +6840,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG Time to Solve an Incident", "title": "AVG Time to Solve an Incident", "description": "", @@ -7198,9 +7168,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made SLA?", "title": "Made SLA?", "description": "", @@ -7560,9 +7528,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-IncidentBreakdownbyPriority", "title": "Tooltip - Incident Breakdown by Priority", "description": "", @@ -7812,9 +7778,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Problems", "title": "Overdue Problems", "description": "", @@ -8252,9 +8216,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-ProblemBreakdownbyPriority", "title": "Tooltip - Problem Breakdown by Priority", "description": "", @@ -8588,9 +8550,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-RequestBreakdownbyPriority", "title": "Tooltip - Request Breakdown by Priority", "description": "", @@ -8950,9 +8910,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age of Active Problems", "title": "Age of Active Problems", "description": "", @@ -9338,9 +9296,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Incidents", "title": "Opened Incidents", "description": "", @@ -9778,9 +9734,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/Workbookpublishedds/Sheet1", "title": "published sheet ds", "description": "", @@ -10091,9 +10045,7 @@ }, { "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "title": "dvd Rental Dashboard", "description": "", "charts": [ @@ -10193,9 +10145,7 @@ }, { "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "title": "Story 1", "description": "", "charts": [], @@ -10293,9 +10243,7 @@ }, { "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "title": "Executive Dashboard", "description": "", "charts": [ @@ -10774,9 +10722,7 @@ "customProperties": { "hasExtracts": "True", "extractLastRefreshTime": "2018-02-09T00:05:25Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-02-09T00:05:25Z", - "type": "EmbeddedDatasource" + "extractLastUpdateTime": "2018-02-09T00:05:25Z" }, "name": "Marketo", "tags": [] @@ -12711,13 +12657,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, + "customProperties": {}, "name": "Customer Payment Query", "tags": [] } @@ -13047,13 +12987,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, + "customProperties": {}, "name": "actor+ (dvdrental)", "tags": [] } @@ -13957,13 +13891,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, + "customProperties": {}, "name": "Superstore Datasource", "tags": [] } @@ -14916,9 +14844,7 @@ "customProperties": { "hasExtracts": "True", "extractLastRefreshTime": "2018-01-18T19:35:39Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-01-18T19:35:39Z", - "type": "EmbeddedDatasource" + "extractLastUpdateTime": "2018-01-18T19:35:39Z" }, "name": "Requests", "tags": [] @@ -21779,9 +21705,7 @@ "customProperties": { "hasExtracts": "True", "extractLastRefreshTime": "2018-01-18T20:21:33Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-01-18T20:21:33Z", - "type": "EmbeddedDatasource" + "extractLastUpdateTime": "2018-01-18T20:21:33Z" }, "name": "Problems", "tags": [] @@ -25890,9 +25814,7 @@ "customProperties": { "hasExtracts": "True", "extractLastRefreshTime": "2018-01-18T20:13:08Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-01-18T20:13:08Z", - "type": "EmbeddedDatasource" + "extractLastUpdateTime": "2018-01-18T20:13:08Z" }, "name": "Incidents", "tags": [] @@ -31232,13 +31154,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, + "customProperties": {}, "name": "test publish datasource", "tags": [] } @@ -31626,13 +31542,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "PublishedDatasource" - }, + "customProperties": {}, "name": "test publish datasource", "description": "description for test publish datasource", "tags": [] @@ -32092,13 +32002,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "PublishedDatasource" - }, + "customProperties": {}, "name": "Superstore Datasource", "description": "Description for Superstore dataset", "tags": [] @@ -33480,7 +33384,6 @@ { "fieldPath": "Test_Variant", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33493,7 +33396,6 @@ { "fieldPath": "Mailing_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33506,7 +33408,6 @@ { "fieldPath": "Campaign_Run_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33519,7 +33420,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33532,7 +33432,6 @@ { "fieldPath": "Activity_Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -33545,7 +33444,6 @@ { "fieldPath": "Choice_Number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33558,7 +33456,6 @@ { "fieldPath": "Step_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33571,7 +33468,6 @@ { "fieldPath": "Campaign_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33584,7 +33480,6 @@ { "fieldPath": "Lead_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33597,7 +33492,6 @@ { "fieldPath": "Has_Predictive", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -33678,7 +33572,6 @@ { "fieldPath": "Campaign_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33691,7 +33584,6 @@ { "fieldPath": "Campaign_Run_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33704,7 +33596,6 @@ { "fieldPath": "Link", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -33717,7 +33608,6 @@ { "fieldPath": "Test_Variant", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33730,7 +33620,6 @@ { "fieldPath": "Platform", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -33743,7 +33632,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33756,7 +33644,6 @@ { "fieldPath": "Activity_Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -33769,7 +33656,6 @@ { "fieldPath": "Choice_Number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33782,7 +33668,6 @@ { "fieldPath": "Mailing_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33795,7 +33680,6 @@ { "fieldPath": "Step_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33808,7 +33692,6 @@ { "fieldPath": "Lead_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33821,7 +33704,6 @@ { "fieldPath": "Link_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -33834,7 +33716,6 @@ { "fieldPath": "Is_Predictive", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -33847,7 +33728,6 @@ { "fieldPath": "Device", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -33860,7 +33740,6 @@ { "fieldPath": "Is_Mobile_Device", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -33873,7 +33752,6 @@ { "fieldPath": "User_Agent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -33954,7 +33832,6 @@ { "fieldPath": "Platform", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -33967,7 +33844,6 @@ { "fieldPath": "Device", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -33980,7 +33856,6 @@ { "fieldPath": "Choice_Number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33993,7 +33868,6 @@ { "fieldPath": "Lead_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34006,7 +33880,6 @@ { "fieldPath": "Activity_Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34019,7 +33892,6 @@ { "fieldPath": "Test_Variant", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34032,7 +33904,6 @@ { "fieldPath": "Is_Mobile_Device", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34045,7 +33916,6 @@ { "fieldPath": "Has_Predictive", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34058,7 +33928,6 @@ { "fieldPath": "Step_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34071,7 +33940,6 @@ { "fieldPath": "User_Agent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34084,7 +33952,6 @@ { "fieldPath": "Mailing_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34097,7 +33964,6 @@ { "fieldPath": "Campaign_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34110,7 +33976,6 @@ { "fieldPath": "Campaign_Run_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34123,7 +33988,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34204,7 +34068,6 @@ { "fieldPath": "Test_Variant", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34217,7 +34080,6 @@ { "fieldPath": "Campaign_Run_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34230,7 +34092,6 @@ { "fieldPath": "Activity_Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34243,7 +34104,6 @@ { "fieldPath": "Mailing_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34256,7 +34116,6 @@ { "fieldPath": "Has_Predictive", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34269,7 +34128,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34282,7 +34140,6 @@ { "fieldPath": "Campaign_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34295,7 +34152,6 @@ { "fieldPath": "Step_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34308,7 +34164,6 @@ { "fieldPath": "Lead_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34321,7 +34176,6 @@ { "fieldPath": "Choice_Number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34402,7 +34256,6 @@ { "fieldPath": "programName", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34415,7 +34268,6 @@ { "fieldPath": "name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34428,7 +34280,6 @@ { "fieldPath": "programId", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34441,7 +34292,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34454,7 +34304,6 @@ { "fieldPath": "createdAt", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34467,7 +34316,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34480,7 +34328,6 @@ { "fieldPath": "workspaceName", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34493,7 +34340,6 @@ { "fieldPath": "updatedAt", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34506,7 +34352,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34587,7 +34432,6 @@ { "fieldPath": "postal_code", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34600,7 +34444,6 @@ { "fieldPath": "last_update", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34613,7 +34456,6 @@ { "fieldPath": "phone", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34626,7 +34468,6 @@ { "fieldPath": "address2", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34639,7 +34480,6 @@ { "fieldPath": "address", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34652,7 +34492,6 @@ { "fieldPath": "city_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34665,7 +34504,6 @@ { "fieldPath": "district", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34678,7 +34516,6 @@ { "fieldPath": "address_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34759,7 +34596,6 @@ { "fieldPath": "last_update", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34772,7 +34608,6 @@ { "fieldPath": "last_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34785,7 +34620,6 @@ { "fieldPath": "first_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34798,7 +34632,6 @@ { "fieldPath": "actor_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34879,7 +34712,6 @@ { "fieldPath": "Person", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34892,7 +34724,6 @@ { "fieldPath": "Region", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34970,7 +34801,6 @@ { "fieldPath": "Returned", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34983,7 +34813,6 @@ { "fieldPath": "Order ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35061,7 +34890,6 @@ { "fieldPath": "Product ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35074,7 +34902,6 @@ { "fieldPath": "Category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35087,7 +34914,6 @@ { "fieldPath": "Postal Code", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35100,7 +34926,6 @@ { "fieldPath": "City", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35113,7 +34938,6 @@ { "fieldPath": "Quantity", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35126,7 +34950,6 @@ { "fieldPath": "State", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35139,7 +34962,6 @@ { "fieldPath": "Order Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -35152,7 +34974,6 @@ { "fieldPath": "Customer Name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35165,7 +34986,6 @@ { "fieldPath": "Country/Region", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35178,7 +34998,6 @@ { "fieldPath": "Sales", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35191,7 +35010,6 @@ { "fieldPath": "Segment", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35204,7 +35022,6 @@ { "fieldPath": "Sub-Category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35217,7 +35034,6 @@ { "fieldPath": "Profit", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35230,7 +35046,6 @@ { "fieldPath": "Product Name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35243,7 +35058,6 @@ { "fieldPath": "Customer ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35256,7 +35070,6 @@ { "fieldPath": "Order ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35269,7 +35082,6 @@ { "fieldPath": "Row ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35282,7 +35094,6 @@ { "fieldPath": "Discount", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35295,7 +35106,6 @@ { "fieldPath": "Ship Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -35308,7 +35118,6 @@ { "fieldPath": "Ship Mode", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35321,7 +35130,6 @@ { "fieldPath": "Region", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35401,7 +35209,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35414,7 +35221,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35427,7 +35233,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35440,7 +35245,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35453,7 +35257,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35466,7 +35269,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35479,7 +35281,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35492,7 +35293,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35505,7 +35305,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35518,7 +35317,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35531,7 +35329,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35544,7 +35341,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -35557,7 +35353,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35570,7 +35365,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -35583,7 +35377,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35596,7 +35389,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35609,7 +35401,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35622,7 +35413,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35635,7 +35425,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35648,7 +35437,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35661,7 +35449,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35674,7 +35461,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35687,7 +35473,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35700,7 +35485,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35713,7 +35497,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35726,7 +35509,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35739,7 +35521,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35752,7 +35533,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35765,7 +35545,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35778,7 +35557,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35791,7 +35569,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35804,7 +35581,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35817,7 +35593,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35830,7 +35605,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35843,7 +35617,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35856,7 +35629,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35869,7 +35641,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35882,7 +35653,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35895,7 +35665,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35908,7 +35677,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35921,7 +35689,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35934,7 +35701,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35947,7 +35713,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35960,7 +35725,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35973,7 +35737,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35986,7 +35749,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -35999,7 +35761,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36012,7 +35773,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36025,7 +35785,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36038,7 +35797,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36051,7 +35809,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36064,7 +35821,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36077,7 +35833,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36090,7 +35845,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36103,7 +35857,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36116,7 +35869,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36129,7 +35881,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36142,7 +35893,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36155,7 +35905,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36168,7 +35917,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36181,7 +35929,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36194,7 +35941,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36275,7 +36021,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36288,7 +36033,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36301,7 +36045,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36314,7 +36057,6 @@ { "fieldPath": "requested_for", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36327,7 +36069,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36340,7 +36081,6 @@ { "fieldPath": "price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36353,7 +36093,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36366,7 +36105,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36379,7 +36117,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36392,7 +36129,6 @@ { "fieldPath": "delivery_address", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36405,7 +36141,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36418,7 +36153,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36431,7 +36165,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36444,7 +36177,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36457,7 +36189,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36470,7 +36201,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -36483,7 +36213,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36496,7 +36225,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36509,7 +36237,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36522,7 +36249,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36535,7 +36261,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36548,7 +36273,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36561,7 +36285,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36574,7 +36297,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36587,7 +36309,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36600,7 +36321,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36613,7 +36333,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36626,7 +36345,6 @@ { "fieldPath": "requested_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -36639,7 +36357,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36652,7 +36369,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36665,7 +36381,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36678,7 +36393,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36691,7 +36405,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36704,7 +36417,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36717,7 +36429,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36730,7 +36441,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36743,7 +36453,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36756,7 +36465,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36769,7 +36477,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36782,7 +36489,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36795,7 +36501,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36808,7 +36513,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36821,7 +36525,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36834,7 +36537,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36847,7 +36549,6 @@ { "fieldPath": "request_state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36860,7 +36561,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36873,7 +36573,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36886,7 +36585,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36899,7 +36597,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -36912,7 +36609,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36925,7 +36621,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36938,7 +36633,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36951,7 +36645,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36964,7 +36657,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36977,7 +36669,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36990,7 +36681,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37003,7 +36693,6 @@ { "fieldPath": "stage", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37016,7 +36705,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37029,7 +36717,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37042,7 +36729,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37055,7 +36741,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37068,7 +36753,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37081,7 +36765,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37094,7 +36777,6 @@ { "fieldPath": "special_instructions", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37107,7 +36789,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37120,7 +36801,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37133,7 +36813,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37146,7 +36825,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37159,7 +36837,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37172,7 +36849,6 @@ { "fieldPath": "calendar_stc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37253,7 +36929,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37266,7 +36941,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37279,7 +36953,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37292,7 +36965,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37305,7 +36977,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37318,7 +36989,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37331,7 +37001,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37344,7 +37013,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37357,7 +37025,6 @@ { "fieldPath": "configuration_item", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37370,7 +37037,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37383,7 +37049,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37396,7 +37061,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37409,7 +37073,6 @@ { "fieldPath": "recurring_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37422,7 +37085,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37435,7 +37097,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37448,7 +37109,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37461,7 +37121,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37474,7 +37133,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37487,7 +37145,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37500,7 +37157,6 @@ { "fieldPath": "sc_catalog", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37513,7 +37169,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37526,7 +37181,6 @@ { "fieldPath": "stage", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37539,7 +37193,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37552,7 +37205,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37565,7 +37217,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37578,7 +37229,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37591,7 +37241,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37604,7 +37253,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37617,7 +37265,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37630,7 +37277,6 @@ { "fieldPath": "price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37643,7 +37289,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37656,7 +37301,6 @@ { "fieldPath": "estimated_delivery", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37669,7 +37313,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37682,7 +37325,6 @@ { "fieldPath": "context", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37695,7 +37337,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37708,7 +37349,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37721,7 +37361,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37734,7 +37373,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37747,7 +37385,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37760,7 +37397,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37773,7 +37409,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37786,7 +37421,6 @@ { "fieldPath": "billable", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37799,7 +37433,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37812,7 +37445,6 @@ { "fieldPath": "recurring_frequency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37825,7 +37457,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37838,7 +37469,6 @@ { "fieldPath": "cat_item", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37851,7 +37481,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37864,7 +37493,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37877,7 +37505,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37890,7 +37517,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37903,7 +37529,6 @@ { "fieldPath": "order_guide", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37916,7 +37541,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37929,7 +37553,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37942,7 +37565,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37955,7 +37577,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37968,7 +37589,6 @@ { "fieldPath": "backordered", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37981,7 +37601,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37994,7 +37613,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38007,7 +37625,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38020,7 +37637,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38033,7 +37649,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38046,7 +37661,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38059,7 +37673,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38072,7 +37685,6 @@ { "fieldPath": "request", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38085,7 +37697,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38098,7 +37709,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38111,7 +37721,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38124,7 +37733,6 @@ { "fieldPath": "quantity", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38137,7 +37745,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38150,7 +37757,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38163,7 +37769,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38176,7 +37781,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38189,7 +37793,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38202,7 +37805,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38215,7 +37817,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38228,7 +37829,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38309,7 +37909,6 @@ { "fieldPath": "sc_catalogs", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38322,7 +37921,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38335,7 +37933,6 @@ { "fieldPath": "type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38348,7 +37945,6 @@ { "fieldPath": "mobile_picture_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38361,7 +37957,6 @@ { "fieldPath": "workflow", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38374,7 +37969,6 @@ { "fieldPath": "sys_customer_update", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38387,7 +37981,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38400,7 +37993,6 @@ { "fieldPath": "visible_standalone", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38413,7 +38005,6 @@ { "fieldPath": "no_quantity", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38426,7 +38017,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38439,7 +38029,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38452,7 +38041,6 @@ { "fieldPath": "sys_scope", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38465,7 +38053,6 @@ { "fieldPath": "template", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38478,7 +38065,6 @@ { "fieldPath": "no_proceed_checkout", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38491,7 +38077,6 @@ { "fieldPath": "billable", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38504,7 +38089,6 @@ { "fieldPath": "name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38517,7 +38101,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38530,7 +38113,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38543,7 +38125,6 @@ { "fieldPath": "meta", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38556,7 +38137,6 @@ { "fieldPath": "ordered_item_link", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38569,7 +38149,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38582,7 +38161,6 @@ { "fieldPath": "sc_ic_version", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38595,7 +38173,6 @@ { "fieldPath": "image", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38608,7 +38185,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38621,7 +38197,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38634,7 +38209,6 @@ { "fieldPath": "sys_policy", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38647,7 +38221,6 @@ { "fieldPath": "roles", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38660,7 +38233,6 @@ { "fieldPath": "picture", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38673,7 +38245,6 @@ { "fieldPath": "list_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38686,7 +38257,6 @@ { "fieldPath": "no_order_now", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38699,7 +38269,6 @@ { "fieldPath": "vendor", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38712,7 +38281,6 @@ { "fieldPath": "sc_ic_item_staging", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38725,7 +38293,6 @@ { "fieldPath": "no_order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38738,7 +38305,6 @@ { "fieldPath": "entitlement_script", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38751,7 +38317,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38764,7 +38329,6 @@ { "fieldPath": "icon", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38777,7 +38341,6 @@ { "fieldPath": "ignore_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38790,7 +38353,6 @@ { "fieldPath": "start_closed", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38803,7 +38365,6 @@ { "fieldPath": "sys_package", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38816,7 +38377,6 @@ { "fieldPath": "group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38829,7 +38389,6 @@ { "fieldPath": "sys_replace_on_upgrade", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38842,7 +38401,6 @@ { "fieldPath": "cost", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38855,7 +38413,6 @@ { "fieldPath": "use_sc_layout", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38868,7 +38425,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38881,7 +38437,6 @@ { "fieldPath": "availability", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38894,7 +38449,6 @@ { "fieldPath": "model", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38907,7 +38461,6 @@ { "fieldPath": "custom_cart", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38920,7 +38473,6 @@ { "fieldPath": "mobile_picture", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38933,7 +38485,6 @@ { "fieldPath": "category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38946,7 +38497,6 @@ { "fieldPath": "no_cart", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38959,7 +38509,6 @@ { "fieldPath": "mobile_hide_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38972,7 +38521,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38985,7 +38533,6 @@ { "fieldPath": "price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38998,7 +38545,6 @@ { "fieldPath": "delivery_time", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39011,7 +38557,6 @@ { "fieldPath": "no_search", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39024,7 +38569,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39037,7 +38581,6 @@ { "fieldPath": "recurring_frequency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39050,7 +38593,6 @@ { "fieldPath": "recurring_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39063,7 +38605,6 @@ { "fieldPath": "delivery_plan_script", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39076,7 +38617,6 @@ { "fieldPath": "visible_bundle", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39089,7 +38629,6 @@ { "fieldPath": "sys_update_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39102,7 +38641,6 @@ { "fieldPath": "sys_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39115,7 +38653,6 @@ { "fieldPath": "visible_guide", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39128,7 +38665,6 @@ { "fieldPath": "preview", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39141,7 +38677,6 @@ { "fieldPath": "omit_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39222,7 +38757,6 @@ { "fieldPath": "u_u", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39235,7 +38769,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39248,7 +38781,6 @@ { "fieldPath": "source", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39261,7 +38793,6 @@ { "fieldPath": "exclude_manager", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39274,7 +38805,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39287,7 +38817,6 @@ { "fieldPath": "cost_center", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39300,7 +38829,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39313,7 +38841,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39326,7 +38853,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39339,7 +38865,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39352,7 +38877,6 @@ { "fieldPath": "u_lucha", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39365,7 +38889,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39378,7 +38901,6 @@ { "fieldPath": "u_lu2", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39391,7 +38913,6 @@ { "fieldPath": "type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39404,7 +38925,6 @@ { "fieldPath": "roles", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39417,7 +38937,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39430,7 +38949,6 @@ { "fieldPath": "default_assignee", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39443,7 +38961,6 @@ { "fieldPath": "email", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39456,7 +38973,6 @@ { "fieldPath": "manager", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39469,7 +38985,6 @@ { "fieldPath": "name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39482,7 +38997,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39495,7 +39009,6 @@ { "fieldPath": "include_members", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39576,7 +39089,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39589,7 +39101,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39602,7 +39113,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39615,7 +39125,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39628,7 +39137,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39641,7 +39149,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39654,7 +39161,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39667,7 +39173,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39680,7 +39185,6 @@ { "fieldPath": "related_incidents", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39693,7 +39197,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39706,7 +39209,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39719,7 +39221,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39732,7 +39233,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39745,7 +39245,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39758,7 +39257,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39771,7 +39269,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39784,7 +39281,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39797,7 +39293,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39810,7 +39305,6 @@ { "fieldPath": "rfc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39823,7 +39317,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39836,7 +39329,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39849,7 +39341,6 @@ { "fieldPath": "problem_state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39862,7 +39353,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39875,7 +39365,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39888,7 +39377,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39901,7 +39389,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39914,7 +39401,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39927,7 +39413,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39940,7 +39425,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39953,7 +39437,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39966,7 +39449,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39979,7 +39461,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39992,7 +39473,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40005,7 +39485,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40018,7 +39497,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40031,7 +39509,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40044,7 +39521,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40057,7 +39533,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40070,7 +39545,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40083,7 +39557,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40096,7 +39569,6 @@ { "fieldPath": "work_around", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40109,7 +39581,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40122,7 +39593,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40135,7 +39605,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40148,7 +39617,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40161,7 +39629,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40174,7 +39641,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40187,7 +39653,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40200,7 +39665,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40213,7 +39677,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40226,7 +39689,6 @@ { "fieldPath": "known_error", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -40239,7 +39701,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40252,7 +39713,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40265,7 +39725,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40278,7 +39737,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40291,7 +39749,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40304,7 +39761,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40317,7 +39773,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40330,7 +39785,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40343,7 +39797,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40356,7 +39809,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40369,7 +39821,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40382,7 +39833,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40395,7 +39845,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40408,7 +39857,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40421,7 +39869,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40434,7 +39881,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40515,7 +39961,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40528,7 +39973,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40541,7 +39985,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40554,7 +39997,6 @@ { "fieldPath": "severity", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40567,7 +40009,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40580,7 +40021,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40593,7 +40033,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40606,7 +40045,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40619,7 +40057,6 @@ { "fieldPath": "parent_incident", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40632,7 +40069,6 @@ { "fieldPath": "subcategory", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40645,7 +40081,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40658,7 +40093,6 @@ { "fieldPath": "caller_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40671,7 +40105,6 @@ { "fieldPath": "resolved_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40684,7 +40117,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40697,7 +40129,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40710,7 +40141,6 @@ { "fieldPath": "resolved_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40723,7 +40153,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40736,7 +40165,6 @@ { "fieldPath": "business_stc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40749,7 +40177,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40762,7 +40189,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40775,7 +40201,6 @@ { "fieldPath": "category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40788,7 +40213,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40801,7 +40225,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40814,7 +40237,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40827,7 +40249,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40840,7 +40261,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40853,7 +40273,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40866,7 +40285,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40879,7 +40297,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40892,7 +40309,6 @@ { "fieldPath": "child_incidents", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40905,7 +40321,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40918,7 +40333,6 @@ { "fieldPath": "incident_state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40931,7 +40345,6 @@ { "fieldPath": "notify", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40944,7 +40357,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40957,7 +40369,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40970,7 +40381,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40983,7 +40393,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40996,7 +40405,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41009,7 +40417,6 @@ { "fieldPath": "problem_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41022,7 +40429,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41035,7 +40441,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41048,7 +40453,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41061,7 +40465,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41074,7 +40477,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41087,7 +40489,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41100,7 +40501,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41113,7 +40513,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41126,7 +40525,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41139,7 +40537,6 @@ { "fieldPath": "caused_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41152,7 +40549,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41165,7 +40561,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41178,7 +40573,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41191,7 +40585,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -41204,7 +40597,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41217,7 +40609,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41230,7 +40621,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41243,7 +40633,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41256,7 +40645,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41269,7 +40657,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41282,7 +40669,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41295,7 +40681,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41308,7 +40693,6 @@ { "fieldPath": "reopen_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41321,7 +40705,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41334,7 +40717,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -41347,7 +40729,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41360,7 +40741,6 @@ { "fieldPath": "calendar_stc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41373,7 +40753,6 @@ { "fieldPath": "rfc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41386,7 +40765,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41399,7 +40777,6 @@ { "fieldPath": "close_code", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41412,7 +40789,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41425,7 +40801,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41438,7 +40813,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41451,7 +40825,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -41464,7 +40837,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41477,7 +40849,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41490,7 +40861,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41503,7 +40873,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41516,7 +40885,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41529,7 +40897,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41610,7 +40977,6 @@ { "fieldPath": "first_discovered", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41623,7 +40989,6 @@ { "fieldPath": "operational_status", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41636,7 +41001,6 @@ { "fieldPath": "last_discovered", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41649,7 +41013,6 @@ { "fieldPath": "cost_cc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41662,7 +41025,6 @@ { "fieldPath": "checked_in", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41675,7 +41037,6 @@ { "fieldPath": "attributes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41688,7 +41049,6 @@ { "fieldPath": "serial_number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41701,7 +41061,6 @@ { "fieldPath": "vendor", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41714,7 +41073,6 @@ { "fieldPath": "ip_address", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41727,7 +41085,6 @@ { "fieldPath": "support_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41740,7 +41097,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41753,7 +41109,6 @@ { "fieldPath": "asset", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41766,7 +41121,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41779,7 +41133,6 @@ { "fieldPath": "supported_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41792,7 +41145,6 @@ { "fieldPath": "invoice_number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41805,7 +41157,6 @@ { "fieldPath": "managed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41818,7 +41169,6 @@ { "fieldPath": "fault_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41831,7 +41181,6 @@ { "fieldPath": "due_in", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41844,7 +41193,6 @@ { "fieldPath": "cost", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41857,7 +41205,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41870,7 +41217,6 @@ { "fieldPath": "justification", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41883,7 +41229,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41896,7 +41241,6 @@ { "fieldPath": "assigned", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41909,7 +41253,6 @@ { "fieldPath": "model_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41922,7 +41265,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41935,7 +41277,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41948,7 +41289,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41961,7 +41301,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41974,7 +41313,6 @@ { "fieldPath": "lease_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41987,7 +41325,6 @@ { "fieldPath": "monitor", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -42000,7 +41337,6 @@ { "fieldPath": "cost_center", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42013,7 +41349,6 @@ { "fieldPath": "subcategory", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42026,7 +41361,6 @@ { "fieldPath": "delivery_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42039,7 +41373,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42052,7 +41385,6 @@ { "fieldPath": "can_print", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -42065,7 +41397,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42078,7 +41409,6 @@ { "fieldPath": "model_number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42091,7 +41421,6 @@ { "fieldPath": "name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42104,7 +41433,6 @@ { "fieldPath": "start_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42117,7 +41445,6 @@ { "fieldPath": "discovery_source", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42130,7 +41457,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42143,7 +41469,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42156,7 +41481,6 @@ { "fieldPath": "category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42169,7 +41493,6 @@ { "fieldPath": "schedule", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42182,7 +41505,6 @@ { "fieldPath": "fqdn", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42195,7 +41517,6 @@ { "fieldPath": "warranty_expiration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -42208,7 +41529,6 @@ { "fieldPath": "owned_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42221,7 +41541,6 @@ { "fieldPath": "asset_tag", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42234,7 +41553,6 @@ { "fieldPath": "manufacturer", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42247,7 +41565,6 @@ { "fieldPath": "purchase_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -42260,7 +41577,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42273,7 +41589,6 @@ { "fieldPath": "department", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42286,7 +41601,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42299,7 +41613,6 @@ { "fieldPath": "checked_out", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42312,7 +41625,6 @@ { "fieldPath": "unverified", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -42325,7 +41637,6 @@ { "fieldPath": "skip_sync", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -42338,7 +41649,6 @@ { "fieldPath": "po_number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42351,7 +41661,6 @@ { "fieldPath": "order_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42364,7 +41673,6 @@ { "fieldPath": "gl_account", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42377,7 +41685,6 @@ { "fieldPath": "maintenance_schedule", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42390,7 +41697,6 @@ { "fieldPath": "install_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42403,7 +41709,6 @@ { "fieldPath": "dns_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42416,7 +41721,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42429,7 +41733,6 @@ { "fieldPath": "mac_address", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42442,7 +41745,6 @@ { "fieldPath": "change_control", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42455,7 +41757,6 @@ { "fieldPath": "install_status", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -42468,7 +41769,6 @@ { "fieldPath": "due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42481,7 +41781,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} diff --git a/metadata-ingestion/tests/integration/tableau/tableau_state_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_state_mces_golden.json deleted file mode 100644 index fb884f79c0977..0000000000000 --- a/metadata-ingestion/tests/integration/tableau/tableau_state_mces_golden.json +++ /dev/null @@ -1,32464 +0,0 @@ -[ -{ - "entityType": "container", - "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"tableau\", \"workbook_id\": \"1f15d897-7f0c-7c59-037a-afa6a9b7c9a9\"}, \"externalUrl\": \"https://do-not-connect/#/site/acryl/workbooks/15995\", \"name\": \"Email Performance by Campaign\", \"description\": \"Description for Email Performance by Campaign\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:tableau\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Workbook\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:jawadqu@gmail.com\", \"type\": \"DATAOWNER\"}], \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", - "changeType": "UPSERT", - "aspectName": "chartUsageStatistics", - "aspect": { - "value": "{\"timestampMillis\": 1638860400000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"viewsCount\": 5}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Name": "", - "Activity Date": "", - "ID": "", - "Program Name": "", - "Active": "", - "Id": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline - Sent", - "title": "Timeline - Sent", - "description": "", - "lastModified": { - "created": { - "time": 1640200234000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1640200234000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Email Performance by Campaign" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Delivery Rate": "formula: ZN([Delivered Email]/[Sent Email])", - "Name": "", - "Id (Activity - Click Email)": "", - "Activity Date": "", - "Clickthrough Rate": "formula: [Clickthrough Emails]/[Delivered Email]", - "Open Rate": "formula: ZN([Opened Email]/[Delivered Email])", - "Sent Email": "formula: COUNTD([Id])", - "Delivered Email": "formula: COUNTD([Id (Activity - Email Delivered)])", - "ID": "", - "Id (Activity - Open Email)": "", - "Clickthrough Emails": "formula: COUNTD([Id (Activity - Click Email)])", - "Click-to-Open": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", - "Opened Email": "formula: COUNTD([Id (Activity - Open Email)])", - "Id (Activity - Email Delivered)": "", - "Program Name": "", - "Active": "", - "Id": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign List", - "title": "Campaign List", - "description": "", - "lastModified": { - "created": { - "time": 1640200234000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1640200234000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Email Performance by Campaign" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Delivery Rate": "formula: ZN([Delivered Email]/[Sent Email])", - "Name": "", - "Id (Activity - Click Email)": "", - "Activity Date": "", - "Clickthrough Rate": "formula: [Clickthrough Emails]/[Delivered Email]", - "Open Rate": "formula: ZN([Opened Email]/[Delivered Email])", - "Measure Names": "", - "Sent Email": "formula: COUNTD([Id])", - "Delivered Email": "formula: COUNTD([Id (Activity - Email Delivered)])", - "ID": "", - "Id (Activity - Open Email)": "", - "Clickthrough Emails": "formula: COUNTD([Id (Activity - Click Email)])", - "Click-to-Open": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", - "Opened Email": "formula: COUNTD([Id (Activity - Open Email)])", - "Id (Activity - Email Delivered)": "", - "Program Name": "", - "Measure Values": "", - "Active": "", - "Id": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Summary", - "title": "Summary", - "description": "", - "lastModified": { - "created": { - "time": 1640200234000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1640200234000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Email Performance by Campaign" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Delivery Rate": "formula: ZN([Delivered Email]/[Sent Email])", - "Name": "", - "Id (Activity - Click Email)": "", - "Activity Date": "", - "Open Rate": "formula: ZN([Opened Email]/[Delivered Email])", - "Sent Email": "formula: COUNTD([Id])", - "Delivered Email": "formula: COUNTD([Id (Activity - Email Delivered)])", - "ID": "", - "Id (Activity - Open Email)": "", - "Clickthrough Emails": "formula: COUNTD([Id (Activity - Click Email)])", - "Click-to-Open": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", - "Opened Email": "formula: COUNTD([Id (Activity - Open Email)])", - "Id (Activity - Email Delivered)": "", - "Program Name": "", - "Id": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile - Sent by Campaign", - "title": "Mobile - Sent by Campaign", - "description": "", - "lastModified": { - "created": { - "time": 1640200234000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1640200234000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Email Performance by Campaign" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", - "changeType": "UPSERT", - "aspectName": "dashboardUsageStatistics", - "aspect": { - "value": "{\"timestampMillis\": 1638860400000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"viewsCount\": 3}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { - "urn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": {}, - "title": "Email Performance by Campaign", - "description": "", - "charts": [ - "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", - "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", - "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", - "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 1640200234000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1640200234000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Email Performance by Campaign" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"tableau\", \"workbook_id\": \"661fabd0-bed6-8610-e066-0694a81a6cea\"}, \"externalUrl\": \"https://do-not-connect/#/site/acryl/workbooks/15619\", \"name\": \"Dvdrental Workbook\", \"description\": \"\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:tableau\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Workbook\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:jawadqu@gmail.com\", \"type\": \"DATAOWNER\"}], \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Custom SQL Query": "" - }, - "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet1", - "title": "Sheet 1", - "description": "", - "lastModified": { - "created": { - "time": 1639772911000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1642199995000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Dvdrental Workbook" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "payment_date": "", - "amount": "", - "Custom SQL Query": "", - "First Name": "", - "customer_id": "", - "rental_id": "", - "Last Name": "" - }, - "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet2", - "title": "Sheet 2", - "description": "", - "lastModified": { - "created": { - "time": 1639773415000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1642199995000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Dvdrental Workbook" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Category": "", - "Segment": "", - "Customer Name": "" - }, - "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet3", - "title": "Sheet 3", - "description": "", - "lastModified": { - "created": { - "time": 1640375456000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1642199995000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Dvdrental Workbook" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:TAGSHEET3" - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { - "urn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": {}, - "title": "dvd Rental Dashboard", - "description": "", - "charts": [ - "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 1639773866000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1642199995000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Dvdrental Workbook" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { - "urn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": {}, - "title": "Story 1", - "description": "", - "charts": [], - "datasets": [], - "lastModified": { - "created": { - "time": 1639773866000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1642199995000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Dvdrental Workbook" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"tableau\", \"workbook_id\": \"6ffa5a7f-d852-78f1-6c6d-20ac23610ebf\"}, \"externalUrl\": \"https://do-not-connect/#/site/acryl/workbooks/15605\", \"name\": \"Executive Dashboard\", \"description\": \"\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:tableau\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Workbook\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:jawadqu@gmail.com\", \"type\": \"DATAOWNER\"}], \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Opened": "", - "Total # Request": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", - "Due date": "", - "Priority": "", - "Overdue": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", - "Active": "", - "Current Year Total Cases": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Total Active Requests": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "Number": "", - "Closed": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Requests", - "title": "Opened Requests", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Opened": "", - "Due date": "", - "Priority": "", - "Name": "", - "Overdue": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", - "Active": "", - "Current Year Total Cases": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Number": "", - "Closed": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top 10 Items by Requests and YoY Change", - "title": "Top 10 Items by Requests and YoY Change", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Overdue": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", - "Number": "", - "Priority": "", - "Due date": "", - "Total # Problems": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", - "Opened": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Closed": "", - "Active": "", - "Current Year Total Cases": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "Total Active Problems": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Problems", - "title": "Opened Problems", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "% of Overdue": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", - "Priority": "", - "Category (Incident)": "", - "Overdue": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", - "Number": "", - "Opened": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Due date": "", - "Active": "", - "Closed": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue", - "title": "Overdue", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Number": "", - "Priority": "", - "Opened": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "% of critical and high priority": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", - "Active": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High and Critical Priority Problems", - "title": "High and Critical Priority Problems", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Priority": "", - "Current Year Total Cases": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "Category (Incident)": "", - "Overdue": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", - "Number": "", - "Opened": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Due date": "", - "Active": "", - "Closed": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total Incidents by Category and YoY Change", - "title": "Total Incidents by Category and YoY Change", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Number": "", - "Priority": "", - "Problems with Related Incidents": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", - "Related Incidents": "", - "Opened": "", - "% of known error": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Known error": "", - "Active": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known Errors", - "title": "Known Errors", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Opened": "", - "Due date": "", - "Priority": "", - "Overdue": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", - "Active": "", - "% of Overdue": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", - "Number": "", - "Closed": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Requests", - "title": "Overdue Requests", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Priority": "", - "Category (Incident)": "", - "Time to Close an Incident (seconds)": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", - "Opened": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Active": "", - "Closed": "", - "Time to Close an Incident": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG Time to Solve an Incident", - "title": "AVG Time to Solve an Incident", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Opened": "", - "% made SLA": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", - "Priority": "", - "Active": "", - "Made SLA": "", - "Number": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made SLA?", - "title": "Made SLA?", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Priority": "", - "Number": "", - "Opened": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Active": "" - }, - "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-IncidentBreakdownbyPriority", - "title": "Tooltip - Incident Breakdown by Priority", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Overdue": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", - "Number": "", - "Priority": "", - "Due date": "", - "% of Overdue": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", - "Opened": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Closed": "", - "Active": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Problems", - "title": "Overdue Problems", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Number": "", - "Priority": "", - "Opened": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Active": "" - }, - "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-ProblemBreakdownbyPriority", - "title": "Tooltip - Problem Breakdown by Priority", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Opened": "", - "Priority": "", - "Active": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Number": "" - }, - "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-RequestBreakdownbyPriority", - "title": "Tooltip - Request Breakdown by Priority", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Number": "", - "Priority": "", - "Time Span Breakdown": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", - "Opened": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Age of Problems": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", - "Active": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age of Active Problems", - "title": "Age of Active Problems", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Priority": "", - "Current Year Total Cases": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "Category (Incident)": "", - "Overdue": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", - "Total Active Incidents": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "Number": "", - "Opened": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Opened Month Tooltip": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", - "Due date": "", - "Active": "", - "Closed": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Incidents", - "title": "Opened Incidents", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { - "urn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": {}, - "title": "Executive Dashboard", - "description": "", - "charts": [ - "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", - "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", - "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", - "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", - "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", - "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", - "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", - "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", - "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", - "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", - "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", - "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", - "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"tableau\", \"workbook_id\": \"bd040833-8f66-22c0-1b51-bd4ccf5eef7c\"}, \"externalUrl\": \"https://do-not-connect/#/site/acryl/workbooks/17904\", \"name\": \"Workbook published ds\", \"description\": \"\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:tableau\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Workbook\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:jawadqu@gmail.com\", \"type\": \"DATAOWNER\"}], \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "staff_last_name": "", - "amount": "", - "customer_first_name": "" - }, - "externalUrl": "https://do-not-connect/#/site/acryl/views/Workbookpublishedds/Sheet1", - "title": "published sheet ds", - "description": "", - "lastModified": { - "created": { - "time": 1641951867000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - }, - "lastModified": { - "time": 1642658093000, - "actor": "urn:li:corpuser:jawadqu@gmail.com" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Workbook published ds" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "True", - "extractLastRefreshTime": "2018-02-09T00:05:25Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-02-09T00:05:25Z", - "type": "EmbeddedDatasource" - }, - "name": "Marketo", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Delivery Rate", - "nullable": false, - "description": "formula: ZN([Delivered Email]/[Sent Email])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Program ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Platform (Activity - Click Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updated At", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Workspace Name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Mailing ID (Activity - Email Delivered)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Campaign ID (Activity - Email Delivered)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Number of Records", - "nullable": false, - "description": "formula: 1", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Campaign Run ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "User Agent (Activity - Click Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Choice Number (Activity - Email Delivered)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Campaign ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Is Predictive", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Campaign ID (Activity - Click Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Activity Date (Activity - Click Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Has Predictive (Activity - Open Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Step ID (Activity - Open Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Id (Activity - Click Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Step ID (Activity - Click Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Activity Date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Clickthrough Rate", - "nullable": false, - "description": "formula: [Clickthrough Emails]/[Delivered Email]", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Open Rate", - "nullable": false, - "description": "formula: ZN([Opened Email]/[Delivered Email])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Measure Names", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Sent Email", - "nullable": false, - "description": "formula: COUNTD([Id])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Delivered Email", - "nullable": false, - "description": "formula: COUNTD([Id (Activity - Email Delivered)])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Lead ID (Activity - Click Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Choice Number", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Is Mobile Device", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Platform", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Mailing ID (Activity - Open Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Lead ID (Activity - Open Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Test Variant (Activity - Email Delivered)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Campaign Run ID (Activity - Email Delivered)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Id (Activity - Open Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Activity Date (Activity - Open Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Campaign Run ID (Activity - Click Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created At", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Clickthrough Emails", - "nullable": false, - "description": "formula: COUNTD([Id (Activity - Click Email)])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Is Mobile Device (Activity - Click Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Mailing ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Device (Activity - Click Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Mailing ID (Activity - Click Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Choice Number (Activity - Click Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Test Variant", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Link ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Click-to-Open", - "nullable": false, - "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Opened Email", - "nullable": false, - "description": "formula: COUNTD([Id (Activity - Open Email)])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Link", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Lead ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Campaign Run ID (Activity - Open Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Opened Non Clicked Emails", - "nullable": false, - "description": "formula: [Opened Email]-[Clickthrough Emails]", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Test Variant (Activity - Open Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Step ID (Activity - Email Delivered)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Non Clickthrough Email", - "nullable": false, - "description": "formula: [Delivered Email]-[Clickthrough Emails]", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Device", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Lead ID (Activity - Email Delivered)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Id (Activity - Email Delivered)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Program Name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Campaign ID (Activity - Open Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Activity Date (Activity - Email Delivered)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Test Variant (Activity - Click Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Has Predictive", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Non Opened Email", - "nullable": false, - "description": "formula: [Delivered Email]-[Opened Email]", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Measure Values", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Has Predictive (Activity - Email Delivered)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "User Agent", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Active", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Step ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Bounceback", - "nullable": false, - "description": "formula: [Sent Email] - [Delivered Email]", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Choice Number (Activity - Open Email)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Embedded Data Source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, - "name": "Customer Payment Query", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "payment_date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:YEAR" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "amount", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Custom SQL Query", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": {} - } - }, - "nativeDataType": "TABLE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "First Name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "customer_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "rental_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Last Name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Embedded Data Source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, - "name": "actor+ (dvdrental)", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Address2", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Last Update", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "address", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": {} - } - }, - "nativeDataType": "TABLE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "District", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "First Name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Address", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Postal Code", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "country, city", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:HIERARCHYFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Phone", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Last Update (Address)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Address Id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Actor Id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "actor", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": {} - } - }, - "nativeDataType": "TABLE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Last Name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "City Id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Embedded Data Source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, - "name": "Superstore Datasource", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Region (People)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Product ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Profit", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Product", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Sub-Category", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Row ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Order ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Customer ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Ship Mode", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "People", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Order ID (Returns)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "City", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Sales", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Region", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Quantity", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Ship Date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Orders", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Profit Ratio", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Returns", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Manufacturer", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Location", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Returned", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Category", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Regional Manager", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Postal Code", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Profit (bin)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Segment", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Top Customers by Profit", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Country/Region", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Discount", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Order Date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Product Name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Customer Name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "State", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Embedded Data Source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "True", - "extractLastRefreshTime": "2018-01-18T19:35:39Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-01-18T19:35:39Z", - "type": "EmbeddedDatasource" - }, - "name": "Requests", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Closed by (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Location", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Opened", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Mobile Picture", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Delivery task", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "% made SLA", - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Meta", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Correlation ID (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Domain Path (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work notes (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Domain", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Short description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Domain Path (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updated by (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Additional comments (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Recurring price", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Duration (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Contact type", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Total # Request", - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - }, - { - "tag": "urn:li:tag:ATTRIBUTE" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Requested for date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Upon approval", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Ordered item link", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Visible elsewhere", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Price", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updated (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Recurring Price Frequency (Catalog Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Correlation display (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Order (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Group list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Update name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Special instructions", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Execution Plan", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Visible on Bundles", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval set (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "No search", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Active (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Configuration item (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Due date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "List Price", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Company (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Additional assignee list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Expected start (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Task type (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Description (Catalog Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Escalation (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Customer update", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Time worked", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Price (Catalog Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Due date (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Order Guide", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Package", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Watch list (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Delivery plan (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Parent (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Urgency", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Delivery address", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Domain (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work notes list (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Estimated Delivery", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Activity due (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Model", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Billable", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Opened by (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Recurring Price Frequency", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Recurring Price", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Fulfillment group", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Additional comments (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work notes (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Parent", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Opened by (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Backordered", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "No cart", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Ignore price", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Follow up (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Number (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval set (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Priority", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Opened (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Correlation ID (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Short description (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Close notes (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Duration (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Overdue", - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Task type", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Location (Catalog Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Measure Names", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Upon approval (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Cart", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Contact type (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Assigned to (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Replace on upgrade", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Impact (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Additional assignee list (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Context", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Order (Catalog Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Priority (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Correlation display (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Requested for", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Configuration item (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "No order", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Billable (Catalog Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created (Catalog Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Upon reject", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Assignment group (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Sys ID (Catalog Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updated", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "No quantity", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Mobile Picture Type", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Quantity", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Delivery plan", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Business duration (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Protection policy", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Follow up", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Location (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Omit price in cart", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Catalogs", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Urgency (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "User input (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updates (Catalog Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Entitlement script", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Follow up (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Company", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work start (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Published version", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Preview link", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updates (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updates", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Closed (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Assignment group (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Comments and Work notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Price (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Active (Catalog Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Stage (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Visible on Guides", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Active", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Configuration item (Requested Item) 1", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Due date (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Assigned to", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Escalation (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updated (Catalog Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Type", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Category", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Delivery time", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Item", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Expected start (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Expected start", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updated (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Stage", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Delivery task (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Request", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Description (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created by (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Active (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Short description (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Correlation display", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Reassignment count (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work end", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Current Year Total Cases", - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - }, - { - "tag": "urn:li:tag:ATTRIBUTE" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Close notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updates (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Closed by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Delivery plan (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Resolve Time", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Business duration", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Max Year?", - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updated by (Catalog Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Task type (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Time worked (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Comments and Work notes (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Time worked (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Contact type (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Additional assignee list (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval history", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "State", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Group list (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Made SLA", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Knowledge (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Icon", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "User input (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Display name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Business service (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Reassignment count", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Number (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Duration", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Escalation", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Comments and Work notes (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Number of Records", - "nullable": false, - "description": "formula: 1", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work start (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Close notes (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "State (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Description (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "State (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "No proceed checkout", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Measure Values", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Watch list (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "SLA due (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Impact (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updated by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Sys ID (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Closed (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Activity due (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Use cart layout", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "No order now", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "SLA due (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Application", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "User input", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Additional comments", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Reassignment count (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Template", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Urgency (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Catalog", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Image", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Knowledge", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Impact", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created by (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Priority (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updated by (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval history (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Made SLA (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "% of Overdue", - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Workflow", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Location (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Class", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created from item design", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Business service", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Upon approval (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Knowledge (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Migrated Data", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": {} - } - }, - "nativeDataType": "TABLE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Parent (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Company (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Assigned to (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Closed by (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Roles", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "SLA due", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Short description (Catalog Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Hide price (mobile listings)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work start", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Business duration (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Assignment group", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Upon reject (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Availability", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Vendor", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work notes list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Picture", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Group list (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work notes list (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Opened by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Watch list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Upon reject (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Delivery plan script", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Order (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work end (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Sys ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Opened (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Made SLA (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval set", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval history (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Sys ID (Requested Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work end (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Total Active Requests", - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - }, - { - "tag": "urn:li:tag:ATTRIBUTE" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Delivery task (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Start closed", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Domain (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Correlation ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Number", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Closed", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Request state", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Business service (Request)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Configuration item", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Activity due", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Cost", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Domain Path", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Order", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created by (Catalog Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Embedded Data Source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "True", - "extractLastRefreshTime": "2018-01-18T20:21:33Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-01-18T20:21:33Z", - "type": "EmbeddedDatasource" - }, - "name": "Problems", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "SLA due (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Reassignment count", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Active (Group)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Expected start", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Expected start (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Impact", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Location (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created by (Group)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Time worked", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Active (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Duration", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Default assignee", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updates", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Workaround", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Overdue", - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Domain Path", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Number", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created by (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Closed by (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval set", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Short description (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updated by (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Sys ID (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work notes (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Escalation", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Migrated Data", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": {} - } - }, - "nativeDataType": "TABLE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Correlation display", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Business service", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Knowledge", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Comments and Work notes (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Delivery task (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Delivery task", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work notes list (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work notes list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Opened by (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Parent (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Escalation (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "u", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Due date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Time Span Breakdown", - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Activity due", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Made SLA (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Business duration", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "u_", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Change request", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Close notes (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Correlation display (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval set (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Upon approval (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Follow up (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updates (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Assigned to (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "% of Overdue", - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Sys ID (Group)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updated (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Time worked (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Assignment group", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "SLA due", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Cost center", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Delivery plan (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Domain (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Activity due (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Group list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Roles", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Total # Problems", - "nullable": false, - "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - }, - { - "tag": "urn:li:tag:ATTRIBUTE" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Additional comments", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Duration (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updated by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Domain Path (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Parent (Group)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Watch list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Due date (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Business duration (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Urgency (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Correlation ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work start (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Contact type", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Urgency", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Number of Records", - "nullable": false, - "description": "formula: 1", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Opened (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Sys ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Manager", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Exclude manager", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Lucha", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Task type", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Made SLA", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Priority (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Description (Group)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Order (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Company", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Problems with Related Incidents", - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Follow up", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "User input (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Parent", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Related Incidents", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Watch list (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updated", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "State (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Closed by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Task type (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Group list (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Opened", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "User input", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Source", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Closed (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Opened by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work start", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Additional assignee list (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "% of known error", - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work end", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Problem state", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Upon approval", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Domain", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Reassignment count (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Contact type (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Upon reject (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Close notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Max Year?", - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Configuration item (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Short description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Company (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Delivery plan", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Additional comments (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Knowledge (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Known error", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Group email", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Age of Problems", - "nullable": false, - "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Priority", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Comments and Work notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Measure Values", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval history", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Assigned to", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Impact (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updated (Group)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "% of critical and high priority", - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Closed", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Configuration item", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Business service (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Additional assignee list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Include members", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Order", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Measure Names", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Location", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Description (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Current Year Total Cases", - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - }, - { - "tag": "urn:li:tag:ATTRIBUTE" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created (Group)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updated by (Group)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Number (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "State", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval history (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Type", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work end (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Upon reject", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Correlation ID (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Assignment group (Problem)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Total Active Problems", - "nullable": false, - "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - }, - { - "tag": "urn:li:tag:ATTRIBUTE" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Active", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updates (Group)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Embedded Data Source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "True", - "extractLastRefreshTime": "2018-01-18T20:13:08Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-01-18T20:13:08Z", - "type": "EmbeddedDatasource" - }, - "name": "Incidents", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Assignment group", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Closed (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updates (Configuration Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Attributes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "User input", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "User input (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Short description (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Close notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Escalation (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Impact", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Close code (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Migrated Data", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": {} - } - }, - "nativeDataType": "TABLE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Delivery task (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updated by (Configuration Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Expected start", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Status", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Notify (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Due", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Closed by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Upon approval", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Requires verification", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Maintenance schedule", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Escalation", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updates (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Parent", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Domain Path (Configuration Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Location (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Upon reject", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Warranty expiration", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "GL account", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Additional assignee list (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Opened by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Additional assignee list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work notes (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Company", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Knowledge", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Made SLA", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "First discovered", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval history (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Asset", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "% of Overdue", - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work start", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Domain", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "SLA due", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Group list (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Priority", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Duration (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Assigned to (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Skip sync", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "DNS Domain", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updated", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Caller (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Department", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Resolved by (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Correlation ID (Configuration Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Opened (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Current Year Total Cases", - "nullable": false, - "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - }, - { - "tag": "urn:li:tag:ATTRIBUTE" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Close notes (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Managed by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Measure Names", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Model number", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created by (Configuration Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Assignment group (Configuration Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Location", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "PO number", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Short description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Business resolve time (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Child Incidents (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "IP Address", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Configuration item", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Order (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Asset tag", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Due in", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updated by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Manufacturer", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Correlation display", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Business duration", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Business service", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Checked out", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Category (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Fully qualified domain name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Installed", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Order", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Purchased", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Lease contract", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Vendor", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Overdue", - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Due date (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Additional comments", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval history", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Category (Configuration Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Parent Incident (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updated (Configuration Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Cost currency", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "SLA due (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Impact (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Subcategory (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Opened by (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Activity due (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Comments and Work notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Cost", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Reassignment count (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Urgency (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Monitor", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Watch list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval set", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Time worked", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Serial number", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Model ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Parent (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Time to Close an Incident (seconds)", - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Owned by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Activity due", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Invoice number", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updated by (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval set (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Duration", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Start date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Ordered", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Assigned to", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Follow up", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Business duration (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Order received", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Discovery source", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Closed by (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work notes list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Total Active Incidents", - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - }, - { - "tag": "urn:li:tag:ATTRIBUTE" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Number", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Class", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Description (Configuration Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Operational status", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Expected start (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work notes list (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Opened", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Resolve time (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Reopen count (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created by (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Assigned to (Configuration Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Most recent discovery", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Domain (Configuration Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Upon reject (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work end", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Company (Configuration Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Knowledge (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Max Year?", - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Location (Configuration Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Watch list (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Delivery task", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Fault count", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Caused by Change (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updated (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Sys ID (Configuration Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "MAC Address", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval group", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Approval", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Priority (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Urgency", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Delivery plan (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Company (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Additional comments (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Business service (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Schedule", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Sys ID (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Supported by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Configuration item (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Task type", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Support group", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Active (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Correlation display (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Justification", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Change Request (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Updates", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Incident state (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Domain (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Made SLA (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Opened Month Tooltip", - "nullable": false, - "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Problem (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Measure Values", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Group list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Checked in", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Severity (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Number of Records", - "nullable": false, - "description": "formula: 1", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Time worked (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Cost center", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work end (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Domain Path", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Due date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Contact type", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Created (Configuration Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Delivery plan", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Subcategory (Configuration Item)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Sys ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Comments and Work notes (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Can Print", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Active", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Correlation ID (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Number (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Follow up (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Task type (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Domain Path (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Closed", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Description (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Reassignment count", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Contact type (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Assignment group (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Comments", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "State", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Work start (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Correlation ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "State (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Resolved (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Assigned", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Time to Close an Incident", - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Upon approval (Incident)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Embedded Data Source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, - "name": "test publish datasource", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "customer_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Custom SQL Query", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "staff_last_name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "staff_first_name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "customer_last_name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "amount", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "customer_first_name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "payment_date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Embedded Data Source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "PublishedDatasource" - }, - "name": "test publish datasource", - "description": "description for test publish datasource", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "payment_date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:YEAR" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "staff_first_name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "customer_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "amount", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Published SQL Query", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": {} - } - }, - "nativeDataType": "TABLE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "customer_last_name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "customer_first_name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "staff_last_name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Published Data Source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/Samples" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "PublishedDatasource" - }, - "name": "Superstore Datasource", - "description": "Description for Superstore dataset", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Top Customers by Profit", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:SETFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Returns", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": {} - } - }, - "nativeDataType": "TABLE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Segment", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Profit Ratio", - "nullable": false, - "description": "formula: SUM([Profit])/SUM([Sales])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "City", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Profit", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Quantity", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Returned", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Category", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Product Name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Orders", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": {} - } - }, - "nativeDataType": "TABLE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Product ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Profit (bin)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:BINFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Order ID (Returns)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Person", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Sub-Category", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Postal Code", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Product", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:HIERARCHYFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Ship Date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:YEAR" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Location", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:HIERARCHYFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "People", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": {} - } - }, - "nativeDataType": "TABLE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Country/Region", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Customer ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Region", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Ship Mode", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Order ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Sales", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Customer Name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Row ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Manufacturer", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:GROUPFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Region (People)", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Discount", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "Order Date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:YEAR" - } - ] - }, - "isPartOfKey": false - }, - { - "fieldPath": "State", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Published Data Source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "NUMERIC", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "last_name", - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "rental_id", - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "I4", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "first_name", - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "payment_date", - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DBTIMESTAMP", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "customer_id", - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "I4", - "recursive": false, - "isPartOfKey": false - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Customer Payment Query" - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": {}, - "name": "Custom SQL Query", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": false, - "viewLogic": "SELECT\n\tcustomer.customer_id,\n\tfirst_name,\n\tlast_name,\n\tamount,\n\tpayment_date,\n\trental_id\nFROM\n\tcustomer\nINNER JOIN payment \n ON payment.customer_id = customer.customer_id\nwhere customer.customer_id = <[Parameters].[Parameter 1]>\nORDER BY payment_date", - "viewLanguage": "SQL" - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"View\", \"Custom SQL\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:tableau" - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "customer_id", - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "I4", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "staff_first_name", - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "amount", - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "NUMERIC", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "customer_first_name", - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "payment_date", - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DBTIMESTAMP", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "staff_last_name", - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "customer_last_name", - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "isPartOfKey": false - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/test publish datasource" - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": {}, - "name": "Custom SQL Query", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": false, - "viewLogic": "SELECT\n\tc.customer_id,\n\tc.first_name customer_first_name,\n\tc.last_name customer_last_name,\n\ts.first_name staff_first_name,\n\ts.last_name staff_last_name,\n\tamount,\n\tpayment_date\nFROM\n\tcustomer c\nINNER JOIN payment p \n ON p.customer_id = c.customer_id\nINNER JOIN staff s \n ON p.staff_id = s.staff_id\nORDER BY payment_date", - "viewLanguage": "SQL" - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"View\", \"Custom SQL\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Test_Variant", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Mailing_ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Campaign_Run_ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Activity_Date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Choice_Number", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Step_ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Campaign_ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Lead_ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Has_Predictive", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Campaign_ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Campaign_Run_ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Link", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Test_Variant", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Platform", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Activity_Date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Choice_Number", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Mailing_ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Step_ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Lead_ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Link_ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Is_Predictive", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Device", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Is_Mobile_Device", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "User_Agent", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Platform", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Device", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Choice_Number", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Lead_ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Activity_Date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Test_Variant", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Is_Mobile_Device", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Has_Predictive", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Step_ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "User_Agent", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Mailing_ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Campaign_ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Campaign_Run_ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Test_Variant", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Campaign_Run_ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Activity_Date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Mailing_ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Has_Predictive", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Campaign_ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Step_ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Lead_ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Choice_Number", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "programName", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "programId", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "createdAt", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "workspaceName", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "updatedAt", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "active", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/actor+ (dvdrental)" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "postal_code", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "last_update", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DBTIMESTAMP", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "phone", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "address2", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "address", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "city_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "I2", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "district", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "address_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "I4", - "recursive": false, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/actor+ (dvdrental)" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "last_update", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DBTIMESTAMP", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "last_name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "first_name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "actor_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "I4", - "recursive": false, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Incidents" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "time_worked", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "work_notes_list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "group_list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "parent", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "expected_start", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "due_date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "work_end", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "cmdb_ci", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "business_duration", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "work_start", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "closed_at", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "user_input", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "reassignment_count", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "approval", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "short_description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "impact", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "knowledge", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "closed_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "calendar_duration", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "delivery_task", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sla_due", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_class_name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "comments", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "upon_reject", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "upon_approval", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "approval_history", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_created_on", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "correlation_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "opened_at", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "approval_set", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "escalation", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "delivery_plan", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "comments_and_work_notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "close_notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "watch_list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "opened_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "activity_due", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_updated_on", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "work_notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_updated_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "assigned_to", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "assignment_group", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "order", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_domain_path", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_mod_count", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "business_service", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "priority", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "correlation_display", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "active", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_domain", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "company", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "urgency", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "number", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "state", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "made_sla", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_created_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "additional_assignee_list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "contact_type", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "location", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "follow_up", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Requests" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "work_notes_list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "closed_at", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "user_input", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "requested_for", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "opened_at", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "price", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "assigned_to", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "number", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "delivery_plan", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "delivery_address", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "business_duration", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "urgency", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "work_end", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "due_date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "parent", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "made_sla", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "assignment_group", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_class_name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "additional_assignee_list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "work_start", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_domain_path", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "time_worked", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "comments_and_work_notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "group_list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "business_service", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "correlation_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "cmdb_ci", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "requested_date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "WDC_DATE", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "follow_up", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_updated_on", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "state", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "comments", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "approval_set", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "close_notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "upon_approval", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "company", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "activity_due", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "contact_type", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "approval", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "calendar_duration", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "reassignment_count", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "delivery_task", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "approval_history", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_created_on", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "request_state", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "watch_list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "upon_reject", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "expected_start", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "active", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "opened_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "work_notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "impact", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sla_due", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "correlation_display", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "priority", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "stage", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_created_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "escalation", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "closed_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "short_description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "location", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "special_instructions", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "order", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_updated_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_mod_count", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "knowledge", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_domain", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "calendar_stc", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Requests" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "watch_list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "due_date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "made_sla", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "parent", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "assigned_to", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_mod_count", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "cmdb_ci", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_domain", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "configuration_item", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "closed_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "active", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "expected_start", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "recurring_price", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "work_end", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "short_description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "approval", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "opened_at", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "order", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "assignment_group", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sc_catalog", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "knowledge", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "stage", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "correlation_display", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "reassignment_count", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "delivery_plan", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_class_name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "user_input", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "activity_due", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "price", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "work_notes_list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "estimated_delivery", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "additional_assignee_list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "context", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "business_duration", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "approval_set", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "priority", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_updated_on", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "state", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "business_service", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "billable", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "approval_history", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "recurring_frequency", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "contact_type", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "cat_item", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_updated_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_domain_path", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "comments", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "impact", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "order_guide", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sla_due", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_created_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "comments_and_work_notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "opened_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "backordered", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "correlation_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "group_list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "delivery_task", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "number", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_created_on", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "company", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "work_start", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "request", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "close_notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "work_notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "calendar_duration", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "quantity", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "follow_up", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "location", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "upon_reject", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "closed_at", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "time_worked", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "escalation", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "urgency", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "upon_approval", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Requests" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "sc_catalogs", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_updated_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "type", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "mobile_picture_type", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "workflow", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_customer_update", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_class_name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "visible_standalone", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "no_quantity", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "order", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_updated_on", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_scope", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "template", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "no_proceed_checkout", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "billable", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "delivery_plan", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "meta", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "ordered_item_link", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_mod_count", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sc_ic_version", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "image", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "short_description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_policy", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "roles", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "picture", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "list_price", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "no_order_now", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "vendor", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sc_ic_item_staging", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "no_order", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "entitlement_script", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "active", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "icon", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "ignore_price", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "start_closed", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_package", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "group", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_replace_on_upgrade", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "cost", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_FLOAT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "use_sc_layout", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "location", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "availability", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "model", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "custom_cart", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "mobile_picture", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "category", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "no_cart", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "mobile_hide_price", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_created_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "price", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "delivery_time", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "no_search", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_created_on", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "recurring_frequency", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "recurring_price", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "delivery_plan_script", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "visible_bundle", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_update_name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "visible_guide", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "preview", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "omit_price", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Problems" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "u_u", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_updated_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "source", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "exclude_manager", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "cost_center", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_created_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "parent", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_mod_count", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_updated_on", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "u_lucha", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "u_lu2", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "type", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "roles", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_created_on", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "default_assignee", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "email", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "manager", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "active", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "include_members", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Problems" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "opened_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_class_name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_created_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_updated_on", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "closed_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "close_notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "made_sla", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "state", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "related_incidents", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "business_duration", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_domain", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "delivery_task", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "priority", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_created_on", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "escalation", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "business_service", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "comments_and_work_notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "rfc", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_domain_path", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "cmdb_ci", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "problem_state", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "delivery_plan", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "user_input", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "active", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "location", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "expected_start", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "calendar_duration", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "number", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sla_due", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "work_notes_list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "knowledge", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_updated_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "time_worked", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "order", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "assignment_group", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "upon_approval", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "company", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "opened_at", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "group_list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "work_around", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "work_end", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "correlation_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "approval_set", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "urgency", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "impact", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "short_description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "approval", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "closed_at", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "known_error", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "due_date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "work_start", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "activity_due", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_mod_count", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "correlation_display", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "contact_type", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "additional_assignee_list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "approval_history", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "reassignment_count", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "follow_up", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "comments", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "work_notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "parent", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "assigned_to", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "watch_list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "upon_reject", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Incidents" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "sys_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "correlation_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "urgency", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "severity", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "business_service", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "location", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "approval_set", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "closed_at", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "parent_incident", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "subcategory", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "company", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "caller_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "resolved_at", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "group_list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "correlation_display", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "resolved_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_created_on", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "business_stc", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_domain_path", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "parent", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "category", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "user_input", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "number", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "escalation", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "state", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "approval_history", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "impact", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "expected_start", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "additional_assignee_list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "child_incidents", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "cmdb_ci", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "incident_state", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "notify", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "work_notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "reassignment_count", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "contact_type", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "opened_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_class_name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "problem_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "due_date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "approval", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "order", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "opened_at", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "work_notes_list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "priority", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "time_worked", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_domain", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "caused_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_updated_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "upon_reject", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "delivery_task", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "knowledge", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_updated_on", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "calendar_duration", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "closed_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "comments", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "short_description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "assigned_to", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "assignment_group", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "work_end", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "reopen_count", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "work_start", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "made_sla", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_mod_count", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "calendar_stc", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "rfc", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "delivery_plan", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "close_code", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "close_notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "activity_due", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_created_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "active", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "business_duration", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "follow_up", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "comments_and_work_notes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "upon_approval", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "watch_list", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sla_due", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Incidents" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "first_discovered", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "operational_status", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "last_discovered", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "cost_cc", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "checked_in", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "attributes", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "serial_number", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "vendor", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "ip_address", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "support_group", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_updated_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "asset", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_domain", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "supported_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "invoice_number", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "managed_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "fault_count", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "due_in", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "cost", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "correlation_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "justification", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_created_on", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "assigned", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "model_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_class_name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "comments", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "company", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "lease_id", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "monitor", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "cost_center", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "subcategory", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "delivery_date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "assignment_group", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "can_print", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "short_description", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "model_number", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "start_date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "discovery_source", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_domain_path", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "assigned_to", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "category", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "schedule", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "fqdn", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "warranty_expiration", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "WDC_DATE", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "owned_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "asset_tag", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "manufacturer", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "purchase_date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "WDC_DATE", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "location", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "department", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_updated_on", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "checked_out", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "unverified", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "skip_sync", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "po_number", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "order_date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "gl_account", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "maintenance_schedule", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "install_date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "dns_domain", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_created_by", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "mac_address", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "change_control", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "install_status", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "due", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "sys_mod_count", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/Samples/Superstore Datasource" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Person", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WSTR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Region", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WSTR", - "recursive": false, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/Samples/Superstore Datasource" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Returned", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WSTR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Order ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WSTR", - "recursive": false, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/Samples/Superstore Datasource" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Product ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WSTR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Category", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WSTR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Postal Code", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "I8", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "City", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WSTR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Quantity", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "I8", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "State", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WSTR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Order Date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Customer Name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WSTR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Country/Region", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WSTR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Sales", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "R8", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Segment", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WSTR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Sub-Category", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WSTR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Profit", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "R8", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Product Name", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WSTR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Customer ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WSTR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Order ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WSTR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Row ID", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "I8", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Discount", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "R8", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Ship Date", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Ship Mode", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WSTR", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Region", - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WSTR", - "recursive": false, - "isPartOfKey": false - } - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/test publish datasource" - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/test publish datasource" - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/test publish datasource" - ] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test" - } -} -] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json index 65e74b1899069..e94ff0118e04c 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json @@ -1039,9 +1039,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign List", "title": "Campaign List", "description": "", @@ -1605,9 +1603,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Summary", "title": "Summary", "description": "", @@ -2223,9 +2219,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile - Sent by Campaign", "title": "Mobile - Sent by Campaign", "description": "", @@ -2737,9 +2731,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet1", "title": "Sheet 1", "description": "", @@ -2887,9 +2879,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet2", "title": "Sheet 2", "description": "", @@ -3205,9 +3195,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet3", "title": "Sheet 3", "description": "", @@ -3407,9 +3395,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Requests", "title": "Opened Requests", "description": "", @@ -3913,9 +3899,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top 10 Items by Requests and YoY Change", "title": "Top 10 Items by Requests and YoY Change", "description": "", @@ -4387,9 +4371,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Problems", "title": "Opened Problems", "description": "", @@ -4893,9 +4875,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue", "title": "Overdue", "description": "", @@ -5280,9 +5260,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High and Critical Priority Problems", "title": "High and Critical Priority Problems", "description": "", @@ -5647,9 +5625,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total Incidents by Category and YoY Change", "title": "Total Incidents by Category and YoY Change", "description": "", @@ -6037,9 +6013,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known Errors", "title": "Known Errors", "description": "", @@ -6482,9 +6456,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Requests", "title": "Overdue Requests", "description": "", @@ -6901,9 +6873,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG Time to Solve an Incident", "title": "AVG Time to Solve an Incident", "description": "", @@ -7236,9 +7206,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made SLA?", "title": "Made SLA?", "description": "", @@ -7603,9 +7571,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-IncidentBreakdownbyPriority", "title": "Tooltip - Incident Breakdown by Priority", "description": "", @@ -7860,9 +7826,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Problems", "title": "Overdue Problems", "description": "", @@ -8305,9 +8269,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-ProblemBreakdownbyPriority", "title": "Tooltip - Problem Breakdown by Priority", "description": "", @@ -8646,9 +8608,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-RequestBreakdownbyPriority", "title": "Tooltip - Request Breakdown by Priority", "description": "", @@ -9013,9 +8973,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age of Active Problems", "title": "Age of Active Problems", "description": "", @@ -9406,9 +9364,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Incidents", "title": "Opened Incidents", "description": "", @@ -9851,9 +9807,7 @@ }, { "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "externalUrl": "https://do-not-connect/#/site/acryl/views/Workbookpublishedds/Sheet1", "title": "published sheet ds", "description": "", @@ -10174,9 +10128,7 @@ }, { "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "title": "dvd Rental Dashboard", "description": "", "charts": [ @@ -10281,9 +10233,7 @@ }, { "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "title": "Story 1", "description": "", "charts": [], @@ -10386,9 +10336,7 @@ }, { "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": { - "luid": "" - }, + "customProperties": {}, "title": "Executive Dashboard", "description": "", "charts": [ @@ -10872,9 +10820,7 @@ "customProperties": { "hasExtracts": "True", "extractLastRefreshTime": "2018-02-09T00:05:25Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-02-09T00:05:25Z", - "type": "EmbeddedDatasource" + "extractLastUpdateTime": "2018-02-09T00:05:25Z" }, "name": "Marketo", "tags": [] @@ -12814,13 +12760,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, + "customProperties": {}, "name": "Customer Payment Query", "tags": [] } @@ -13155,13 +13095,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, + "customProperties": {}, "name": "actor+ (dvdrental)", "tags": [] } @@ -14070,13 +14004,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, + "customProperties": {}, "name": "Superstore Datasource", "tags": [] } @@ -15034,9 +14962,7 @@ "customProperties": { "hasExtracts": "True", "extractLastRefreshTime": "2018-01-18T19:35:39Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-01-18T19:35:39Z", - "type": "EmbeddedDatasource" + "extractLastUpdateTime": "2018-01-18T19:35:39Z" }, "name": "Requests", "tags": [] @@ -21902,9 +21828,7 @@ "customProperties": { "hasExtracts": "True", "extractLastRefreshTime": "2018-01-18T20:21:33Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-01-18T20:21:33Z", - "type": "EmbeddedDatasource" + "extractLastUpdateTime": "2018-01-18T20:21:33Z" }, "name": "Problems", "tags": [] @@ -26018,9 +25942,7 @@ "customProperties": { "hasExtracts": "True", "extractLastRefreshTime": "2018-01-18T20:13:08Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-01-18T20:13:08Z", - "type": "EmbeddedDatasource" + "extractLastUpdateTime": "2018-01-18T20:13:08Z" }, "name": "Incidents", "tags": [] @@ -31365,13 +31287,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, + "customProperties": {}, "name": "test publish datasource", "tags": [] } @@ -31764,13 +31680,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "PublishedDatasource" - }, + "customProperties": {}, "name": "test publish datasource", "description": "description for test publish datasource", "tags": [] @@ -32235,13 +32145,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "PublishedDatasource" - }, + "customProperties": {}, "name": "Superstore Datasource", "description": "Description for Superstore dataset", "tags": [] @@ -33621,7 +33525,6 @@ { "fieldPath": "Test_Variant", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33634,7 +33537,6 @@ { "fieldPath": "Mailing_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33647,7 +33549,6 @@ { "fieldPath": "Campaign_Run_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33660,7 +33561,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33673,7 +33573,6 @@ { "fieldPath": "Activity_Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -33686,7 +33585,6 @@ { "fieldPath": "Choice_Number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33699,7 +33597,6 @@ { "fieldPath": "Step_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33712,7 +33609,6 @@ { "fieldPath": "Campaign_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33725,7 +33621,6 @@ { "fieldPath": "Lead_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33738,7 +33633,6 @@ { "fieldPath": "Has_Predictive", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -33823,7 +33717,6 @@ { "fieldPath": "Campaign_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33836,7 +33729,6 @@ { "fieldPath": "Campaign_Run_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33849,7 +33741,6 @@ { "fieldPath": "Link", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -33862,7 +33753,6 @@ { "fieldPath": "Test_Variant", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33875,7 +33765,6 @@ { "fieldPath": "Platform", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -33888,7 +33777,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33901,7 +33789,6 @@ { "fieldPath": "Activity_Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -33914,7 +33801,6 @@ { "fieldPath": "Choice_Number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33927,7 +33813,6 @@ { "fieldPath": "Mailing_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33940,7 +33825,6 @@ { "fieldPath": "Step_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33953,7 +33837,6 @@ { "fieldPath": "Lead_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -33966,7 +33849,6 @@ { "fieldPath": "Link_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -33979,7 +33861,6 @@ { "fieldPath": "Is_Predictive", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -33992,7 +33873,6 @@ { "fieldPath": "Device", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34005,7 +33885,6 @@ { "fieldPath": "Is_Mobile_Device", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34018,7 +33897,6 @@ { "fieldPath": "User_Agent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34103,7 +33981,6 @@ { "fieldPath": "Platform", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34116,7 +33993,6 @@ { "fieldPath": "Device", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34129,7 +34005,6 @@ { "fieldPath": "Choice_Number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34142,7 +34017,6 @@ { "fieldPath": "Lead_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34155,7 +34029,6 @@ { "fieldPath": "Activity_Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34168,7 +34041,6 @@ { "fieldPath": "Test_Variant", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34181,7 +34053,6 @@ { "fieldPath": "Is_Mobile_Device", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34194,7 +34065,6 @@ { "fieldPath": "Has_Predictive", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34207,7 +34077,6 @@ { "fieldPath": "Step_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34220,7 +34089,6 @@ { "fieldPath": "User_Agent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34233,7 +34101,6 @@ { "fieldPath": "Mailing_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34246,7 +34113,6 @@ { "fieldPath": "Campaign_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34259,7 +34125,6 @@ { "fieldPath": "Campaign_Run_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34272,7 +34137,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34357,7 +34221,6 @@ { "fieldPath": "Test_Variant", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34370,7 +34233,6 @@ { "fieldPath": "Campaign_Run_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34383,7 +34245,6 @@ { "fieldPath": "Activity_Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34396,7 +34257,6 @@ { "fieldPath": "Mailing_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34409,7 +34269,6 @@ { "fieldPath": "Has_Predictive", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34422,7 +34281,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34435,7 +34293,6 @@ { "fieldPath": "Campaign_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34448,7 +34305,6 @@ { "fieldPath": "Step_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34461,7 +34317,6 @@ { "fieldPath": "Lead_ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34474,7 +34329,6 @@ { "fieldPath": "Choice_Number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34559,7 +34413,6 @@ { "fieldPath": "programName", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34572,7 +34425,6 @@ { "fieldPath": "name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34585,7 +34437,6 @@ { "fieldPath": "programId", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34598,7 +34449,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34611,7 +34461,6 @@ { "fieldPath": "createdAt", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34624,7 +34473,6 @@ { "fieldPath": "id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34637,7 +34485,6 @@ { "fieldPath": "workspaceName", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34650,7 +34497,6 @@ { "fieldPath": "updatedAt", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34663,7 +34509,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -34748,7 +34593,6 @@ { "fieldPath": "postal_code", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34761,7 +34605,6 @@ { "fieldPath": "last_update", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34774,7 +34617,6 @@ { "fieldPath": "phone", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34787,7 +34629,6 @@ { "fieldPath": "address2", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34800,7 +34641,6 @@ { "fieldPath": "address", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34813,7 +34653,6 @@ { "fieldPath": "city_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34826,7 +34665,6 @@ { "fieldPath": "district", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34839,7 +34677,6 @@ { "fieldPath": "address_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -34924,7 +34761,6 @@ { "fieldPath": "last_update", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -34937,7 +34773,6 @@ { "fieldPath": "last_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34950,7 +34785,6 @@ { "fieldPath": "first_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -34963,7 +34797,6 @@ { "fieldPath": "actor_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35048,7 +34881,6 @@ { "fieldPath": "Person", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35061,7 +34893,6 @@ { "fieldPath": "Region", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35143,7 +34974,6 @@ { "fieldPath": "Returned", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35156,7 +34986,6 @@ { "fieldPath": "Order ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35238,7 +35067,6 @@ { "fieldPath": "Product ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35251,7 +35079,6 @@ { "fieldPath": "Category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35264,7 +35091,6 @@ { "fieldPath": "Postal Code", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35277,7 +35103,6 @@ { "fieldPath": "City", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35290,7 +35115,6 @@ { "fieldPath": "Quantity", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35303,7 +35127,6 @@ { "fieldPath": "State", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35316,7 +35139,6 @@ { "fieldPath": "Order Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -35329,7 +35151,6 @@ { "fieldPath": "Customer Name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35342,7 +35163,6 @@ { "fieldPath": "Country/Region", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35355,7 +35175,6 @@ { "fieldPath": "Sales", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35368,7 +35187,6 @@ { "fieldPath": "Segment", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35381,7 +35199,6 @@ { "fieldPath": "Sub-Category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35394,7 +35211,6 @@ { "fieldPath": "Profit", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35407,7 +35223,6 @@ { "fieldPath": "Product Name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35420,7 +35235,6 @@ { "fieldPath": "Customer ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35433,7 +35247,6 @@ { "fieldPath": "Order ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35446,7 +35259,6 @@ { "fieldPath": "Row ID", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35459,7 +35271,6 @@ { "fieldPath": "Discount", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35472,7 +35283,6 @@ { "fieldPath": "Ship Date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -35485,7 +35295,6 @@ { "fieldPath": "Ship Mode", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35498,7 +35307,6 @@ { "fieldPath": "Region", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35582,7 +35390,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35595,7 +35402,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35608,7 +35414,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35621,7 +35426,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35634,7 +35438,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35647,7 +35450,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35660,7 +35462,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35673,7 +35474,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35686,7 +35486,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35699,7 +35498,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35712,7 +35510,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35725,7 +35522,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -35738,7 +35534,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35751,7 +35546,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -35764,7 +35558,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35777,7 +35570,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35790,7 +35582,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35803,7 +35594,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35816,7 +35606,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35829,7 +35618,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35842,7 +35630,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35855,7 +35642,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35868,7 +35654,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35881,7 +35666,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35894,7 +35678,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35907,7 +35690,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35920,7 +35702,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -35933,7 +35714,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35946,7 +35726,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35959,7 +35738,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35972,7 +35750,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -35985,7 +35762,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -35998,7 +35774,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36011,7 +35786,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36024,7 +35798,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36037,7 +35810,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36050,7 +35822,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36063,7 +35834,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36076,7 +35846,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36089,7 +35858,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36102,7 +35870,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36115,7 +35882,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36128,7 +35894,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36141,7 +35906,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36154,7 +35918,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36167,7 +35930,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -36180,7 +35942,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36193,7 +35954,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36206,7 +35966,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36219,7 +35978,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36232,7 +35990,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36245,7 +36002,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36258,7 +36014,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36271,7 +36026,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36284,7 +36038,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36297,7 +36050,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36310,7 +36062,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36323,7 +36074,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36336,7 +36086,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36349,7 +36098,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36362,7 +36110,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36375,7 +36122,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36460,7 +36206,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36473,7 +36218,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36486,7 +36230,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36499,7 +36242,6 @@ { "fieldPath": "requested_for", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36512,7 +36254,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36525,7 +36266,6 @@ { "fieldPath": "price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36538,7 +36278,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36551,7 +36290,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36564,7 +36302,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36577,7 +36314,6 @@ { "fieldPath": "delivery_address", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36590,7 +36326,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36603,7 +36338,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36616,7 +36350,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36629,7 +36362,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36642,7 +36374,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36655,7 +36386,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -36668,7 +36398,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36681,7 +36410,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36694,7 +36422,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36707,7 +36434,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36720,7 +36446,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36733,7 +36458,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36746,7 +36470,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36759,7 +36482,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36772,7 +36494,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36785,7 +36506,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36798,7 +36518,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36811,7 +36530,6 @@ { "fieldPath": "requested_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -36824,7 +36542,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36837,7 +36554,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36850,7 +36566,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36863,7 +36578,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36876,7 +36590,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36889,7 +36602,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36902,7 +36614,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36915,7 +36626,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36928,7 +36638,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36941,7 +36650,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36954,7 +36662,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -36967,7 +36674,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -36980,7 +36686,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -36993,7 +36698,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37006,7 +36710,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37019,7 +36722,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37032,7 +36734,6 @@ { "fieldPath": "request_state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37045,7 +36746,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37058,7 +36758,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37071,7 +36770,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37084,7 +36782,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37097,7 +36794,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37110,7 +36806,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37123,7 +36818,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37136,7 +36830,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37149,7 +36842,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37162,7 +36854,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37175,7 +36866,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37188,7 +36878,6 @@ { "fieldPath": "stage", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37201,7 +36890,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37214,7 +36902,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37227,7 +36914,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37240,7 +36926,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37253,7 +36938,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37266,7 +36950,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37279,7 +36962,6 @@ { "fieldPath": "special_instructions", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37292,7 +36974,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37305,7 +36986,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37318,7 +36998,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37331,7 +37010,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37344,7 +37022,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37357,7 +37034,6 @@ { "fieldPath": "calendar_stc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37442,7 +37118,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37455,7 +37130,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37468,7 +37142,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37481,7 +37154,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37494,7 +37166,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37507,7 +37178,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37520,7 +37190,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37533,7 +37202,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37546,7 +37214,6 @@ { "fieldPath": "configuration_item", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37559,7 +37226,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37572,7 +37238,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37585,7 +37250,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37598,7 +37262,6 @@ { "fieldPath": "recurring_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37611,7 +37274,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37624,7 +37286,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37637,7 +37298,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37650,7 +37310,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37663,7 +37322,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37676,7 +37334,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37689,7 +37346,6 @@ { "fieldPath": "sc_catalog", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37702,7 +37358,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37715,7 +37370,6 @@ { "fieldPath": "stage", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37728,7 +37382,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37741,7 +37394,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37754,7 +37406,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37767,7 +37418,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37780,7 +37430,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37793,7 +37442,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37806,7 +37454,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37819,7 +37466,6 @@ { "fieldPath": "price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37832,7 +37478,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37845,7 +37490,6 @@ { "fieldPath": "estimated_delivery", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37858,7 +37502,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37871,7 +37514,6 @@ { "fieldPath": "context", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37884,7 +37526,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37897,7 +37538,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37910,7 +37550,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37923,7 +37562,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -37936,7 +37574,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37949,7 +37586,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -37962,7 +37598,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -37975,7 +37610,6 @@ { "fieldPath": "billable", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -37988,7 +37622,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38001,7 +37634,6 @@ { "fieldPath": "recurring_frequency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38014,7 +37646,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38027,7 +37658,6 @@ { "fieldPath": "cat_item", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38040,7 +37670,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38053,7 +37682,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38066,7 +37694,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38079,7 +37706,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38092,7 +37718,6 @@ { "fieldPath": "order_guide", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38105,7 +37730,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38118,7 +37742,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38131,7 +37754,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38144,7 +37766,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38157,7 +37778,6 @@ { "fieldPath": "backordered", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38170,7 +37790,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38183,7 +37802,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38196,7 +37814,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38209,7 +37826,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38222,7 +37838,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38235,7 +37850,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38248,7 +37862,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38261,7 +37874,6 @@ { "fieldPath": "request", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38274,7 +37886,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38287,7 +37898,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38300,7 +37910,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38313,7 +37922,6 @@ { "fieldPath": "quantity", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38326,7 +37934,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38339,7 +37946,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38352,7 +37958,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38365,7 +37970,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38378,7 +37982,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38391,7 +37994,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38404,7 +38006,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38417,7 +38018,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38502,7 +38102,6 @@ { "fieldPath": "sc_catalogs", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38515,7 +38114,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38528,7 +38126,6 @@ { "fieldPath": "type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38541,7 +38138,6 @@ { "fieldPath": "mobile_picture_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38554,7 +38150,6 @@ { "fieldPath": "workflow", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38567,7 +38162,6 @@ { "fieldPath": "sys_customer_update", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38580,7 +38174,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38593,7 +38186,6 @@ { "fieldPath": "visible_standalone", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38606,7 +38198,6 @@ { "fieldPath": "no_quantity", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38619,7 +38210,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38632,7 +38222,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -38645,7 +38234,6 @@ { "fieldPath": "sys_scope", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38658,7 +38246,6 @@ { "fieldPath": "template", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38671,7 +38258,6 @@ { "fieldPath": "no_proceed_checkout", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38684,7 +38270,6 @@ { "fieldPath": "billable", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38697,7 +38282,6 @@ { "fieldPath": "name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38710,7 +38294,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38723,7 +38306,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38736,7 +38318,6 @@ { "fieldPath": "meta", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38749,7 +38330,6 @@ { "fieldPath": "ordered_item_link", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38762,7 +38342,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38775,7 +38354,6 @@ { "fieldPath": "sc_ic_version", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -38788,7 +38366,6 @@ { "fieldPath": "image", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38801,7 +38378,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38814,7 +38390,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38827,7 +38402,6 @@ { "fieldPath": "sys_policy", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38840,7 +38414,6 @@ { "fieldPath": "roles", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38853,7 +38426,6 @@ { "fieldPath": "picture", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38866,7 +38438,6 @@ { "fieldPath": "list_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38879,7 +38450,6 @@ { "fieldPath": "no_order_now", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38892,7 +38462,6 @@ { "fieldPath": "vendor", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38905,7 +38474,6 @@ { "fieldPath": "sc_ic_item_staging", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38918,7 +38486,6 @@ { "fieldPath": "no_order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38931,7 +38498,6 @@ { "fieldPath": "entitlement_script", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38944,7 +38510,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38957,7 +38522,6 @@ { "fieldPath": "icon", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -38970,7 +38534,6 @@ { "fieldPath": "ignore_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38983,7 +38546,6 @@ { "fieldPath": "start_closed", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -38996,7 +38558,6 @@ { "fieldPath": "sys_package", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39009,7 +38570,6 @@ { "fieldPath": "group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39022,7 +38582,6 @@ { "fieldPath": "sys_replace_on_upgrade", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39035,7 +38594,6 @@ { "fieldPath": "cost", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39048,7 +38606,6 @@ { "fieldPath": "use_sc_layout", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39061,7 +38618,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39074,7 +38630,6 @@ { "fieldPath": "availability", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39087,7 +38642,6 @@ { "fieldPath": "model", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39100,7 +38654,6 @@ { "fieldPath": "custom_cart", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39113,7 +38666,6 @@ { "fieldPath": "mobile_picture", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39126,7 +38678,6 @@ { "fieldPath": "category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39139,7 +38690,6 @@ { "fieldPath": "no_cart", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39152,7 +38702,6 @@ { "fieldPath": "mobile_hide_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39165,7 +38714,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39178,7 +38726,6 @@ { "fieldPath": "price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39191,7 +38738,6 @@ { "fieldPath": "delivery_time", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39204,7 +38750,6 @@ { "fieldPath": "no_search", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39217,7 +38762,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39230,7 +38774,6 @@ { "fieldPath": "recurring_frequency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39243,7 +38786,6 @@ { "fieldPath": "recurring_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39256,7 +38798,6 @@ { "fieldPath": "delivery_plan_script", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39269,7 +38810,6 @@ { "fieldPath": "visible_bundle", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39282,7 +38822,6 @@ { "fieldPath": "sys_update_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39295,7 +38834,6 @@ { "fieldPath": "sys_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39308,7 +38846,6 @@ { "fieldPath": "visible_guide", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39321,7 +38858,6 @@ { "fieldPath": "preview", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39334,7 +38870,6 @@ { "fieldPath": "omit_price", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39419,7 +38954,6 @@ { "fieldPath": "u_u", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39432,7 +38966,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39445,7 +38978,6 @@ { "fieldPath": "source", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39458,7 +38990,6 @@ { "fieldPath": "exclude_manager", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39471,7 +39002,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39484,7 +39014,6 @@ { "fieldPath": "cost_center", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39497,7 +39026,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39510,7 +39038,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39523,7 +39050,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39536,7 +39062,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39549,7 +39074,6 @@ { "fieldPath": "u_lucha", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39562,7 +39086,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39575,7 +39098,6 @@ { "fieldPath": "u_lu2", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39588,7 +39110,6 @@ { "fieldPath": "type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39601,7 +39122,6 @@ { "fieldPath": "roles", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39614,7 +39134,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39627,7 +39146,6 @@ { "fieldPath": "default_assignee", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39640,7 +39158,6 @@ { "fieldPath": "email", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39653,7 +39170,6 @@ { "fieldPath": "manager", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39666,7 +39182,6 @@ { "fieldPath": "name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39679,7 +39194,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39692,7 +39206,6 @@ { "fieldPath": "include_members", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39777,7 +39290,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39790,7 +39302,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39803,7 +39314,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39816,7 +39326,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39829,7 +39338,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39842,7 +39350,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39855,7 +39362,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -39868,7 +39374,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39881,7 +39386,6 @@ { "fieldPath": "related_incidents", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39894,7 +39398,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39907,7 +39410,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39920,7 +39422,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39933,7 +39434,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39946,7 +39446,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -39959,7 +39458,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39972,7 +39470,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -39985,7 +39482,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -39998,7 +39494,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40011,7 +39506,6 @@ { "fieldPath": "rfc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40024,7 +39518,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40037,7 +39530,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40050,7 +39542,6 @@ { "fieldPath": "problem_state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40063,7 +39554,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40076,7 +39566,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40089,7 +39578,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -40102,7 +39590,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40115,7 +39602,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40128,7 +39614,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40141,7 +39626,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40154,7 +39638,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40167,7 +39650,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40180,7 +39662,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -40193,7 +39674,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40206,7 +39686,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40219,7 +39698,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40232,7 +39710,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40245,7 +39722,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40258,7 +39734,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40271,7 +39746,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40284,7 +39758,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40297,7 +39770,6 @@ { "fieldPath": "work_around", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40310,7 +39782,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40323,7 +39794,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40336,7 +39806,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40349,7 +39818,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40362,7 +39830,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40375,7 +39842,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40388,7 +39854,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40401,7 +39866,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40414,7 +39878,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40427,7 +39890,6 @@ { "fieldPath": "known_error", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -40440,7 +39902,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40453,7 +39914,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40466,7 +39926,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40479,7 +39938,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40492,7 +39950,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40505,7 +39962,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40518,7 +39974,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40531,7 +39986,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40544,7 +39998,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40557,7 +40010,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40570,7 +40022,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40583,7 +40034,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40596,7 +40046,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40609,7 +40058,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40622,7 +40070,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40635,7 +40082,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40720,7 +40166,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40733,7 +40178,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40746,7 +40190,6 @@ { "fieldPath": "urgency", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40759,7 +40202,6 @@ { "fieldPath": "severity", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40772,7 +40214,6 @@ { "fieldPath": "business_service", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40785,7 +40226,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40798,7 +40238,6 @@ { "fieldPath": "approval_set", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40811,7 +40250,6 @@ { "fieldPath": "closed_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40824,7 +40262,6 @@ { "fieldPath": "parent_incident", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40837,7 +40274,6 @@ { "fieldPath": "subcategory", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40850,7 +40286,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40863,7 +40298,6 @@ { "fieldPath": "caller_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40876,7 +40310,6 @@ { "fieldPath": "resolved_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40889,7 +40322,6 @@ { "fieldPath": "group_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40902,7 +40334,6 @@ { "fieldPath": "correlation_display", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40915,7 +40346,6 @@ { "fieldPath": "resolved_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40928,7 +40358,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -40941,7 +40370,6 @@ { "fieldPath": "business_stc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -40954,7 +40382,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40967,7 +40394,6 @@ { "fieldPath": "parent", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40980,7 +40406,6 @@ { "fieldPath": "category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -40993,7 +40418,6 @@ { "fieldPath": "user_input", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41006,7 +40430,6 @@ { "fieldPath": "number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41019,7 +40442,6 @@ { "fieldPath": "escalation", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41032,7 +40454,6 @@ { "fieldPath": "state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41045,7 +40466,6 @@ { "fieldPath": "approval_history", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41058,7 +40478,6 @@ { "fieldPath": "impact", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41071,7 +40490,6 @@ { "fieldPath": "expected_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41084,7 +40502,6 @@ { "fieldPath": "additional_assignee_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41097,7 +40514,6 @@ { "fieldPath": "child_incidents", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41110,7 +40526,6 @@ { "fieldPath": "cmdb_ci", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41123,7 +40538,6 @@ { "fieldPath": "incident_state", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41136,7 +40550,6 @@ { "fieldPath": "notify", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41149,7 +40562,6 @@ { "fieldPath": "work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41162,7 +40574,6 @@ { "fieldPath": "reassignment_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41175,7 +40586,6 @@ { "fieldPath": "contact_type", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41188,7 +40598,6 @@ { "fieldPath": "opened_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41201,7 +40610,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41214,7 +40622,6 @@ { "fieldPath": "problem_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41227,7 +40634,6 @@ { "fieldPath": "due_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41240,7 +40646,6 @@ { "fieldPath": "approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41253,7 +40658,6 @@ { "fieldPath": "description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41266,7 +40670,6 @@ { "fieldPath": "order", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41279,7 +40682,6 @@ { "fieldPath": "opened_at", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41292,7 +40694,6 @@ { "fieldPath": "work_notes_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41305,7 +40706,6 @@ { "fieldPath": "priority", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41318,7 +40718,6 @@ { "fieldPath": "time_worked", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41331,7 +40730,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41344,7 +40742,6 @@ { "fieldPath": "caused_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41357,7 +40754,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41370,7 +40766,6 @@ { "fieldPath": "upon_reject", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41383,7 +40778,6 @@ { "fieldPath": "delivery_task", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41396,7 +40790,6 @@ { "fieldPath": "knowledge", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -41409,7 +40802,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41422,7 +40814,6 @@ { "fieldPath": "calendar_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41435,7 +40826,6 @@ { "fieldPath": "closed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41448,7 +40838,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41461,7 +40850,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41474,7 +40862,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41487,7 +40874,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41500,7 +40886,6 @@ { "fieldPath": "work_end", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41513,7 +40898,6 @@ { "fieldPath": "reopen_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41526,7 +40910,6 @@ { "fieldPath": "work_start", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41539,7 +40922,6 @@ { "fieldPath": "made_sla", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -41552,7 +40934,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41565,7 +40946,6 @@ { "fieldPath": "calendar_stc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41578,7 +40958,6 @@ { "fieldPath": "rfc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41591,7 +40970,6 @@ { "fieldPath": "delivery_plan", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41604,7 +40982,6 @@ { "fieldPath": "close_code", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41617,7 +40994,6 @@ { "fieldPath": "close_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41630,7 +41006,6 @@ { "fieldPath": "activity_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41643,7 +41018,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41656,7 +41030,6 @@ { "fieldPath": "active", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -41669,7 +41042,6 @@ { "fieldPath": "business_duration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41682,7 +41054,6 @@ { "fieldPath": "follow_up", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41695,7 +41066,6 @@ { "fieldPath": "comments_and_work_notes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41708,7 +41078,6 @@ { "fieldPath": "upon_approval", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41721,7 +41090,6 @@ { "fieldPath": "watch_list", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41734,7 +41102,6 @@ { "fieldPath": "sla_due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41819,7 +41186,6 @@ { "fieldPath": "first_discovered", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41832,7 +41198,6 @@ { "fieldPath": "operational_status", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -41845,7 +41210,6 @@ { "fieldPath": "last_discovered", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41858,7 +41222,6 @@ { "fieldPath": "cost_cc", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41871,7 +41234,6 @@ { "fieldPath": "checked_in", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -41884,7 +41246,6 @@ { "fieldPath": "attributes", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41897,7 +41258,6 @@ { "fieldPath": "serial_number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41910,7 +41270,6 @@ { "fieldPath": "vendor", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41923,7 +41282,6 @@ { "fieldPath": "ip_address", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41936,7 +41294,6 @@ { "fieldPath": "support_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41949,7 +41306,6 @@ { "fieldPath": "sys_updated_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41962,7 +41318,6 @@ { "fieldPath": "asset", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41975,7 +41330,6 @@ { "fieldPath": "sys_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -41988,7 +41342,6 @@ { "fieldPath": "supported_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42001,7 +41354,6 @@ { "fieldPath": "invoice_number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42014,7 +41366,6 @@ { "fieldPath": "managed_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42027,7 +41378,6 @@ { "fieldPath": "fault_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -42040,7 +41390,6 @@ { "fieldPath": "due_in", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42053,7 +41402,6 @@ { "fieldPath": "cost", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42066,7 +41414,6 @@ { "fieldPath": "correlation_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42079,7 +41426,6 @@ { "fieldPath": "justification", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42092,7 +41438,6 @@ { "fieldPath": "sys_created_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42105,7 +41450,6 @@ { "fieldPath": "assigned", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42118,7 +41462,6 @@ { "fieldPath": "model_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42131,7 +41474,6 @@ { "fieldPath": "sys_class_name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42144,7 +41486,6 @@ { "fieldPath": "comments", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42157,7 +41498,6 @@ { "fieldPath": "sys_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42170,7 +41510,6 @@ { "fieldPath": "company", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42183,7 +41522,6 @@ { "fieldPath": "lease_id", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42196,7 +41534,6 @@ { "fieldPath": "monitor", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -42209,7 +41546,6 @@ { "fieldPath": "cost_center", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42222,7 +41558,6 @@ { "fieldPath": "subcategory", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42235,7 +41570,6 @@ { "fieldPath": "delivery_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42248,7 +41582,6 @@ { "fieldPath": "assignment_group", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42261,7 +41594,6 @@ { "fieldPath": "can_print", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -42274,7 +41606,6 @@ { "fieldPath": "short_description", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42287,7 +41618,6 @@ { "fieldPath": "model_number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42300,7 +41630,6 @@ { "fieldPath": "name", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42313,7 +41642,6 @@ { "fieldPath": "start_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42326,7 +41654,6 @@ { "fieldPath": "discovery_source", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42339,7 +41666,6 @@ { "fieldPath": "sys_domain_path", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42352,7 +41678,6 @@ { "fieldPath": "assigned_to", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42365,7 +41690,6 @@ { "fieldPath": "category", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42378,7 +41702,6 @@ { "fieldPath": "schedule", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42391,7 +41714,6 @@ { "fieldPath": "fqdn", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42404,7 +41726,6 @@ { "fieldPath": "warranty_expiration", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -42417,7 +41738,6 @@ { "fieldPath": "owned_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42430,7 +41750,6 @@ { "fieldPath": "asset_tag", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42443,7 +41762,6 @@ { "fieldPath": "manufacturer", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42456,7 +41774,6 @@ { "fieldPath": "purchase_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -42469,7 +41786,6 @@ { "fieldPath": "location", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42482,7 +41798,6 @@ { "fieldPath": "department", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42495,7 +41810,6 @@ { "fieldPath": "sys_updated_on", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42508,7 +41822,6 @@ { "fieldPath": "checked_out", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42521,7 +41834,6 @@ { "fieldPath": "unverified", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -42534,7 +41846,6 @@ { "fieldPath": "skip_sync", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -42547,7 +41858,6 @@ { "fieldPath": "po_number", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42560,7 +41870,6 @@ { "fieldPath": "order_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42573,7 +41882,6 @@ { "fieldPath": "gl_account", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42586,7 +41894,6 @@ { "fieldPath": "maintenance_schedule", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42599,7 +41906,6 @@ { "fieldPath": "install_date", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42612,7 +41918,6 @@ { "fieldPath": "dns_domain", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42625,7 +41930,6 @@ { "fieldPath": "sys_created_by", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42638,7 +41942,6 @@ { "fieldPath": "mac_address", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42651,7 +41954,6 @@ { "fieldPath": "change_control", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -42664,7 +41966,6 @@ { "fieldPath": "install_status", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -42677,7 +41978,6 @@ { "fieldPath": "due", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -42690,7 +41990,6 @@ { "fieldPath": "sys_mod_count", "nullable": false, - "description": "", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} From 99f1624ce719305e8000d903d600034c8b726e25 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 27 Jul 2023 04:36:04 -0700 Subject: [PATCH 191/222] docs(ingest/lookml): clarify connection map config (#8508) --- .../docs/sources/looker/lookml_pre.md | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/metadata-ingestion/docs/sources/looker/lookml_pre.md b/metadata-ingestion/docs/sources/looker/lookml_pre.md index e6528c89cf43f..d78a30fe6ec37 100644 --- a/metadata-ingestion/docs/sources/looker/lookml_pre.md +++ b/metadata-ingestion/docs/sources/looker/lookml_pre.md @@ -7,19 +7,29 @@ To use LookML ingestion through the UI, or automate github checkout through the In a nutshell, there are three steps: 1. Generate a private-public ssh key pair. This will typically generate two files, e.g. looker_datahub_deploy_key (this is the private key) and looker_datahub_deploy_key.pub (this is the public key) -![Image](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/gitssh/ssh-key-generation.png) + ![Image](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/gitssh/ssh-key-generation.png) 2. Add the public key to your Looker git repo as a deploy key with read access (no need to provision write access). Follow the guide [here](https://docs.github.com/en/developers/overview/managing-deploy-keys#deploy-keys) for that. -![Image](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/gitssh/git-deploy-key.png) + ![Image](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/gitssh/git-deploy-key.png) 3. Make note of the private key file, you will need to paste the contents of the file into the **GitHub Deploy Key** field later while setting up [ingestion using the UI](#ui-based-ingestion-recommended-for-ease-of-use). +### Setup your connection mapping + +The connection mapping enables DataHub to accurately generate lineage to your upstream warehouse. +It maps Looker connection names to the platform and database that they're pointing to. + +There's two ways to configure this: + +1. Provide Looker **admin** API credentials, and we'll automatically map lineage correctly. Details on how to do this are below. +2. Manually populate the `connection_to_platform_map` and `project_name` configuration fields. See the starter recipe for an example of what this should look like. + #### [Optional] Create an API key with admin privileges See the [Looker authentication docs](https://docs.looker.com/reference/api-and-integration/api-auth#authentication_with_an_sdk) for the steps to create a client ID and secret. You need to ensure that the API key is attached to a user that has Admin privileges. -If that is not possible, read the configuration section and provide an offline specification of the `connection_to_platform_map` and the `project_name`. +If you don't want to provide admin API credentials, you can manually populate the `connection_to_platform_map` and `project_name` in the ingestion configuration. ### Ingestion Options @@ -80,7 +90,6 @@ on: release: types: [published, edited] workflow_dispatch: - jobs: lookml-metadata-upload: @@ -89,12 +98,13 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: "3.10" - name: Run LookML ingestion run: | pip install 'acryl-datahub[lookml,datahub-rest]' cat << EOF > lookml_ingestion.yml - # LookML ingestion configuration + # LookML ingestion configuration. + # This is a full ingestion recipe, and supports all config options that the LookML source supports. source: type: "lookml" config: @@ -106,8 +116,8 @@ jobs: # Options #connection_to_platform_map: # connection-name: - #platform: platform-name (e.g. snowflake) - #default_db: default-db-name (e.g. DEMO_PIPELINE) + # platform: platform-name (e.g. snowflake) + # default_db: default-db-name (e.g. DEMO_PIPELINE) api: client_id: ${LOOKER_CLIENT_ID} client_secret: ${LOOKER_CLIENT_SECRET} @@ -115,13 +125,13 @@ jobs: sink: type: datahub-rest config: - server: ${DATAHUB_GMS_HOST} - token: ${DATAHUB_TOKEN} + server: ${DATAHUB_GMS_URL} + token: ${DATAHUB_GMS_TOKEN} EOF datahub ingest -c lookml_ingestion.yml env: - DATAHUB_GMS_HOST: ${{ secrets.DATAHUB_GMS_HOST }} - DATAHUB_TOKEN: ${{ secrets.DATAHUB_TOKEN }} + DATAHUB_GMS_URL: ${{ secrets.DATAHUB_GMS_URL }} + DATAHUB_GMS_TOKEN: ${{ secrets.DATAHUB_GMS_TOKEN }} LOOKER_BASE_URL: ${{ secrets.LOOKER_BASE_URL }} LOOKER_CLIENT_ID: ${{ secrets.LOOKER_CLIENT_ID }} LOOKER_CLIENT_SECRET: ${{ secrets.LOOKER_CLIENT_SECRET }} From 3640291493872dd8d05f9399f9cddefc96543a09 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Fri, 28 Jul 2023 14:09:44 -0500 Subject: [PATCH 192/222] config(ebean): add ebean retry configuration (#8500) --- .github/workflows/docker-unified.yml | 17 +++++++++++++++++ docker/build.gradle | 2 ++ docker/docker-compose-with-cassandra.yml | 2 +- docker/docker-compose-without-neo4j.yml | 2 +- docker/docker-compose.yml | 2 +- .../docker-compose-m1.quickstart.yml | 2 +- ...ker-compose-without-neo4j-m1.quickstart.yml | 2 +- ...docker-compose-without-neo4j.quickstart.yml | 2 +- .../quickstart/docker-compose.quickstart.yml | 2 +- docs/cli.md | 1 + .../metadata/entity/EntityServiceImpl.java | 18 +++++++++++++++++- .../factory/entity/EntityServiceFactory.java | 6 +++++- .../cypress/e2e/mutations/deprecations.js | 4 ++-- .../cypress/e2e/mutations/managed_ingestion.js | 3 ++- 14 files changed, 53 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index a23e5a134d294..65028fd260807 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -490,6 +490,9 @@ jobs: if: ${{ needs.setup.outputs.publish != 'true' }} with: image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }} + - name: Disable datahub-actions + run: | + yq -i 'del(.services.datahub-actions)' docker/quickstart/docker-compose-without-neo4j.quickstart.yml - name: run quickstart env: DATAHUB_TELEMETRY_ENABLED: false @@ -501,6 +504,20 @@ jobs: # we are doing this because gms takes time to get ready # and we don't have a better readiness check when bootstrap is done sleep 60s + - name: Disable ES Disk Threshold + run: | + curl -XPUT "http://localhost:9200/_cluster/settings" \ + -H 'Content-Type: application/json' -d'{ + "persistent": { + "cluster": { + "routing": { + "allocation.disk.threshold_enabled": false + } + } + } + }' + - name: Remove Source Code + run: find ./*/* ! -path "./metadata-ingestion*" ! -path "./smoke-test*" ! -path "./gradle*" -delete - name: Smoke test env: RUN_QUICKSTART: false diff --git a/docker/build.gradle b/docker/build.gradle index 16f7b46467cc6..f33e06f383240 100644 --- a/docker/build.gradle +++ b/docker/build.gradle @@ -35,6 +35,8 @@ task quickstart(type: Exec, dependsOn: ':metadata-ingestion:install') { environment "DATAHUB_TELEMETRY_ENABLED", "false" environment "DOCKER_COMPOSE_BASE", "file://${rootProject.projectDir}" + environment "ACTIONS_VERSION", 'alpine3.17-slim' + environment "DATAHUB_ACTIONS_IMAGE", 'nginx' def cmd = [ 'source ../metadata-ingestion/venv/bin/activate && ', diff --git a/docker/docker-compose-with-cassandra.yml b/docker/docker-compose-with-cassandra.yml index 87cd104cd9cd5..1a5f1e208c881 100644 --- a/docker/docker-compose-with-cassandra.yml +++ b/docker/docker-compose-with-cassandra.yml @@ -24,7 +24,7 @@ services: datahub-actions: container_name: datahub-actions hostname: actions - image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} + image: ${DATAHUB_ACTIONS_IMAGE:-acryldata/datahub-actions}:${ACTIONS_VERSION:-head} env_file: datahub-actions/env/docker.env depends_on: datahub-gms: diff --git a/docker/docker-compose-without-neo4j.yml b/docker/docker-compose-without-neo4j.yml index 15d327847936a..26fdee2b6410c 100644 --- a/docker/docker-compose-without-neo4j.yml +++ b/docker/docker-compose-without-neo4j.yml @@ -25,7 +25,7 @@ services: datahub-actions: container_name: datahub-actions hostname: actions - image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} + image: ${DATAHUB_ACTIONS_IMAGE:-acryldata/datahub-actions}:${ACTIONS_VERSION:-head} env_file: datahub-actions/env/docker.env depends_on: datahub-gms: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index f242cb730a68f..5115f90819ef3 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -24,7 +24,7 @@ services: datahub-actions: container_name: datahub-actions hostname: actions - image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} + image: ${DATAHUB_ACTIONS_IMAGE:-acryldata/datahub-actions}:${ACTIONS_VERSION:-head} env_file: datahub-actions/env/docker.env depends_on: datahub-gms: diff --git a/docker/quickstart/docker-compose-m1.quickstart.yml b/docker/quickstart/docker-compose-m1.quickstart.yml index 5ee039b1ceb42..89569510bb001 100644 --- a/docker/quickstart/docker-compose-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-m1.quickstart.yml @@ -45,7 +45,7 @@ services: - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 - SCHEMA_REGISTRY_URL=http://schema-registry:8081 hostname: actions - image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} + image: ${DATAHUB_ACTIONS_IMAGE:-acryldata/datahub-actions}:${ACTIONS_VERSION:-head} datahub-frontend-react: container_name: datahub-frontend-react depends_on: diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index 2ad4f33c6dd99..102500c8f1878 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -45,7 +45,7 @@ services: - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 - SCHEMA_REGISTRY_URL=http://schema-registry:8081 hostname: actions - image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} + image: ${DATAHUB_ACTIONS_IMAGE:-acryldata/datahub-actions}:${ACTIONS_VERSION:-head} datahub-frontend-react: container_name: datahub-frontend-react depends_on: diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index f86d9756ef855..5fa2838255d9a 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -45,7 +45,7 @@ services: - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 - SCHEMA_REGISTRY_URL=http://schema-registry:8081 hostname: actions - image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} + image: ${DATAHUB_ACTIONS_IMAGE:-acryldata/datahub-actions}:${ACTIONS_VERSION:-head} datahub-frontend-react: container_name: datahub-frontend-react depends_on: diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index 2587a76433cdb..adee2da9276aa 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -45,7 +45,7 @@ services: - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 - SCHEMA_REGISTRY_URL=http://schema-registry:8081 hostname: actions - image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} + image: ${DATAHUB_ACTIONS_IMAGE:-acryldata/datahub-actions}:${ACTIONS_VERSION:-head} datahub-frontend-react: container_name: datahub-frontend-react depends_on: diff --git a/docs/cli.md b/docs/cli.md index 0bff921ddd7e3..64b7e2d76bba2 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -121,6 +121,7 @@ The environment variables listed below take precedence over the DataHub CLI conf - `DATAHUB_DEBUG` (default `false`) - Set to `true` to enable debug logging for CLI. Can also be achieved through `--debug` option of the CLI. - `DATAHUB_VERSION` (default `head`) - Set to a specific version to run quickstart with the particular version of docker images. - `ACTIONS_VERSION` (default `head`) - Set to a specific version to run quickstart with that image tag of `datahub-actions` container. +- `DATAHUB_ACTIONS_IMAGE` (default `acryldata/datahub-actions`) - Set to `-slim` to run a slimmer actions container without pyspark/deequ features. ```shell DATAHUB_SKIP_CONFIG=false diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java index 32c77b66679f9..c7a9895992d90 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java @@ -146,7 +146,7 @@ public class EntityServiceImpl implements EntityService { OBJECT_MAPPER.getFactory().setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(maxSize).build()); } - private static final int DEFAULT_MAX_TRANSACTION_RETRY = 3; + private static final int DEFAULT_MAX_TRANSACTION_RETRY = 4; protected final AspectDao _aspectDao; private final EventProducer _producer; @@ -163,6 +163,8 @@ public class EntityServiceImpl implements EntityService { // TODO(iprentic): Move this to a common utils location once used in other places private static final String DELIMITER_SEPARATOR = "␟"; + private final Integer ebeanMaxTransactionRetry; + public EntityServiceImpl( @Nonnull final AspectDao aspectDao, @Nonnull final EventProducer producer, @@ -170,6 +172,17 @@ public EntityServiceImpl( final boolean alwaysEmitChangeLog, final UpdateIndicesService updateIndicesService, final PreProcessHooks preProcessHooks) { + this(aspectDao, producer, entityRegistry, alwaysEmitChangeLog, updateIndicesService, preProcessHooks, DEFAULT_MAX_TRANSACTION_RETRY); + } + + public EntityServiceImpl( + @Nonnull final AspectDao aspectDao, + @Nonnull final EventProducer producer, + @Nonnull final EntityRegistry entityRegistry, + final boolean alwaysEmitChangeLog, + final UpdateIndicesService updateIndicesService, + final PreProcessHooks preProcessHooks, + final Integer retry) { _aspectDao = aspectDao; _producer = producer; @@ -178,8 +191,11 @@ public EntityServiceImpl( _alwaysEmitChangeLog = alwaysEmitChangeLog; _updateIndicesService = updateIndicesService; _preProcessHooks = preProcessHooks; + ebeanMaxTransactionRetry = retry != null ? retry : DEFAULT_MAX_TRANSACTION_RETRY; } + + /** * Retrieves the latest aspects corresponding to a batch of {@link Urn}s based on a provided * set of aspect names. diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityServiceFactory.java index b8edd6918dc6c..5122be69982f0 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityServiceFactory.java @@ -14,6 +14,7 @@ import org.apache.avro.generic.IndexedRecord; import org.apache.kafka.clients.producer.Producer; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.DependsOn; @@ -24,6 +25,9 @@ @Configuration public class EntityServiceFactory { + @Value("${EBEAN_MAX_TRANSACTION_RETRY:#{null}}") + private Integer _ebeanMaxTransactionRetry; + @Bean(name = "entityService") @DependsOn({"entityAspectDao", "kafkaEventProducer", "kafkaHealthChecker", TopicConventionFactory.TOPIC_CONVENTION_BEAN, "entityRegistry"}) @@ -40,6 +44,6 @@ protected EntityService createInstance( final KafkaEventProducer eventProducer = new KafkaEventProducer(producer, convention, kafkaHealthChecker); FeatureFlags featureFlags = configurationProvider.getFeatureFlags(); return new EntityServiceImpl(aspectDao, eventProducer, entityRegistry, - featureFlags.isAlwaysEmitChangeLog(), updateIndicesService, featureFlags.getPreProcessHooks()); + featureFlags.isAlwaysEmitChangeLog(), updateIndicesService, featureFlags.getPreProcessHooks(), _ebeanMaxTransactionRetry); } } diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/deprecations.js b/smoke-test/tests/cypress/cypress/e2e/mutations/deprecations.js index de1b220922102..1d41d155440e8 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutations/deprecations.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutations/deprecations.js @@ -10,10 +10,10 @@ describe("deprecation", () => { cy.addViaFormModal("test deprecation", "Add Deprecation Details"); cy.goToDataset(urn, datasetName); - cy.contains("Deprecated"); + cy.contains("DEPRECATED"); cy.openThreeDotDropdown(); cy.clickOptionWithText("Mark as un-deprecated"); - cy.ensureTextNotPresent("Deprecated"); + cy.ensureTextNotPresent("DEPRECATED"); }); }); diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js b/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js index fd6703a7ef29e..c4d4f76698703 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js @@ -31,7 +31,8 @@ describe("run managed ingestion", () => { cy.waitTextVisible(testName) cy.contains(testName).parent().within(() => { - cy.contains("Succeeded", {timeout: 30000}) + // TODO: Skipping until disk size resolved + // cy.contains("Succeeded", {timeout: 30000}) cy.clickOptionWithTestId("delete-button"); }) cy.clickOptionWithText("Yes") From 9718505fc787a0d064a3d37616389648963c4462 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Fri, 28 Jul 2023 13:09:32 -0700 Subject: [PATCH 193/222] fix(ingest): respect max_threads for ingestion reporter (#8521) --- .../src/datahub/ingestion/graph/client.py | 5 ----- ...atahub_ingestion_checkpointing_provider.py | 4 +++- .../unit/stateful_ingestion/test_configs.py | 20 +++++++++++-------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/graph/client.py b/metadata-ingestion/src/datahub/ingestion/graph/client.py index de8b28d4b95a8..033fd2a68f80b 100644 --- a/metadata-ingestion/src/datahub/ingestion/graph/client.py +++ b/metadata-ingestion/src/datahub/ingestion/graph/client.py @@ -15,7 +15,6 @@ from datahub.cli.cli_utils import get_url_and_token from datahub.configuration.common import ConfigModel, GraphError, OperationalError -from datahub.configuration.validate_field_removal import pydantic_removed_field from datahub.emitter.aspect import TIMESERIES_ASPECT_MAP from datahub.emitter.mce_builder import DEFAULT_ENV, Aspect, make_data_platform_urn from datahub.emitter.mcp import MetadataChangeProposalWrapper @@ -65,10 +64,6 @@ class DatahubClientConfig(ConfigModel): ca_certificate_path: Optional[str] = None disable_ssl_verification: bool = False - _max_threads_moved_to_sink = pydantic_removed_field( - "max_threads", print_warning=False - ) - # Alias for backwards compatibility. # DEPRECATION: Remove in v0.10.2. diff --git a/metadata-ingestion/src/datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider.py b/metadata-ingestion/src/datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider.py index db9caa9706f19..874ee08cc78f9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider.py @@ -42,7 +42,9 @@ def create( elif config_dict is None: raise ConfigurationError("Missing provider configuration.") else: - provider_config = DatahubIngestionStateProviderConfig.parse_obj(config_dict) + provider_config = ( + DatahubIngestionStateProviderConfig.parse_obj_allow_extras(config_dict) + ) if provider_config.datahub_api: graph = DataHubGraph(provider_config.datahub_api) return cls(graph, name) diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/test_configs.py b/metadata-ingestion/tests/unit/stateful_ingestion/test_configs.py index 4f540f81490ac..9edfe8c4a957b 100644 --- a/metadata-ingestion/tests/unit/stateful_ingestion/test_configs.py +++ b/metadata-ingestion/tests/unit/stateful_ingestion/test_configs.py @@ -19,7 +19,7 @@ "token": "dummy_test_tok", "timeout_sec": 10, "extra_headers": {}, - "max_threads": 15, + "max_threads": 10, }, "simple": {}, "default": {}, @@ -44,11 +44,15 @@ "datahub_api": datahub_client_configs["full"], }, DatahubIngestionStateProviderConfig( - datahub_api=DatahubClientConfig( - server="http://localhost:8080", - token="dummy_test_tok", - timeout_sec=10, - extra_headers={}, + # This test verifies that the max_threads arg is ignored. + datahub_api=DatahubClientConfig.parse_obj_allow_extras( + dict( + server="http://localhost:8080", + token="dummy_test_tok", + timeout_sec=10, + extra_headers={}, + max_threads=10, + ) ), ), False, @@ -188,7 +192,7 @@ def test_state_provider_configs( if raises_exception: with pytest.raises(ValidationError): assert expected is None - config_class.parse_obj(config_dict) + config_class.parse_obj_allow_extras(config_dict) else: - config = config_class.parse_obj(config_dict) + config = config_class.parse_obj_allow_extras(config_dict) assert config == expected From 89f23d3c362ab9523a9689bf7e0390c0d3333f2e Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Fri, 28 Jul 2023 13:10:19 -0700 Subject: [PATCH 194/222] chore(ingest): bump sqllineage and sqlparse (#8481) --- metadata-ingestion/setup.py | 9 ++++----- metadata-ingestion/tests/unit/test_usage_common.py | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 04bc03a236f07..5498c96d91af6 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -126,12 +126,11 @@ def get_long_description(): } sqllineage_lib = { - "sqllineage==1.3.6", + "sqllineage==1.3.8", # We don't have a direct dependency on sqlparse but it is a dependency of sqllineage. - # As per https://github.com/reata/sqllineage/issues/361 - # and https://github.com/reata/sqllineage/pull/360 - # sqllineage has compat issues with sqlparse 0.4.4. - "sqlparse==0.4.3", + # There have previously been issues from not pinning sqlparse, so it's best to pin it. + # Related: https://github.com/reata/sqllineage/issues/361 and https://github.com/reata/sqllineage/pull/360 + "sqlparse==0.4.4", } sqlglot_lib = { diff --git a/metadata-ingestion/tests/unit/test_usage_common.py b/metadata-ingestion/tests/unit/test_usage_common.py index 1dbd5b2563cdc..8c9c25593afa8 100644 --- a/metadata-ingestion/tests/unit/test_usage_common.py +++ b/metadata-ingestion/tests/unit/test_usage_common.py @@ -199,7 +199,7 @@ def test_make_usage_workunit(): def test_query_formatting(): test_email = "test_email@test.com" test_query = "select * from foo where id in (select id from bar);" - formatted_test_query: str = "SELECT *\n FROM foo\n WHERE id in (\n SELECT id\n FROM bar\n );" + formatted_test_query: str = "SELECT *\n FROM foo\n WHERE id IN (\n SELECT id\n FROM bar\n );" event_time = datetime(2020, 1, 1) floored_ts = get_time_bucket(event_time, BucketDuration.DAY) From fa05aae959b0f56c6535150a5455f65364e1f1e7 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Fri, 28 Jul 2023 17:41:58 -0500 Subject: [PATCH 195/222] fix(search): fix lightning cache enable logic (#8522) --- .../metadata/search/LineageSearchService.java | 17 ++++--- .../metadata/search/utils/SearchUtils.java | 5 +- .../search/LineageSearchServiceTest.java | 49 ++++++++++++++----- 3 files changed, 47 insertions(+), 24 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java index 5fb1ab0889e71..930a55c56cb6e 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java @@ -189,11 +189,14 @@ public LineageSearchResult searchAcrossLineage(@Nonnull Urn sourceUrn, @Nonnull long numEntities = 0; String codePath = null; try { - if (canDoLightning(lineageRelationships, input, inputFilters, sortCriterion)) { + Filter reducedFilters = + SearchUtils.removeCriteria(inputFilters, criterion -> criterion.getField().equals(DEGREE_FILTER_INPUT)); + + if (canDoLightning(lineageRelationships, input, reducedFilters, sortCriterion)) { codePath = "lightning"; // use lightning approach to return lineage search results LineageSearchResult lineageSearchResult = getLightningSearchResult(lineageRelationships, - inputFilters, from, size, new HashSet<>(entities)); + reducedFilters, from, size, new HashSet<>(entities)); if (!lineageSearchResult.getEntities().isEmpty()) { log.debug("Lightning Lineage entity result: {}", lineageSearchResult.getEntities().get(0).toString()); } @@ -202,7 +205,7 @@ public LineageSearchResult searchAcrossLineage(@Nonnull Urn sourceUrn, @Nonnull } else { codePath = "tortoise"; LineageSearchResult lineageSearchResult = getSearchResultInBatches(lineageRelationships, input, - inputFilters, sortCriterion, from, size, finalFlags); + reducedFilters, sortCriterion, from, size, finalFlags); if (!lineageSearchResult.getEntities().isEmpty()) { log.debug("Lineage entity result: {}", lineageSearchResult.getEntities().get(0).toString()); } @@ -513,15 +516,13 @@ private Filter buildFilter(@Nonnull Set urns, @Nullable Filter inputFilters if (inputFilters == null) { return QueryUtils.newFilter(urnMatchCriterion); } - Filter reducedFilters = - SearchUtils.removeCriteria(inputFilters, criterion -> criterion.getField().equals(DEGREE_FILTER_INPUT)); // Add urn match criterion to each or clause - if (!CollectionUtils.isEmpty(reducedFilters.getOr())) { - for (ConjunctiveCriterion conjunctiveCriterion : reducedFilters.getOr()) { + if (!CollectionUtils.isEmpty(inputFilters.getOr())) { + for (ConjunctiveCriterion conjunctiveCriterion : inputFilters.getOr()) { conjunctiveCriterion.getAnd().add(urnMatchCriterion); } - return reducedFilters; + return inputFilters; } return QueryUtils.newFilter(urnMatchCriterion); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/SearchUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/SearchUtils.java index 38bdef5bd3bdc..5b9ce7444ae38 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/SearchUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/SearchUtils.java @@ -120,9 +120,8 @@ public static String readResourceFile(@Nonnull Class clazz, @Nonnull String file } } - @Nonnull - public static Filter removeCriteria(@Nonnull Filter originalFilter, Predicate shouldRemove) { - if (originalFilter.getOr() != null) { + public static Filter removeCriteria(@Nullable Filter originalFilter, Predicate shouldRemove) { + if (originalFilter != null && originalFilter.getOr() != null) { return new Filter().setOr(new ConjunctiveCriterionArray(originalFilter.getOr() .stream() .map(criteria -> removeCriteria(criteria, shouldRemove)) diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchServiceTest.java index e8872691772de..e5396ffaebe99 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchServiceTest.java @@ -132,7 +132,7 @@ private void resetService(boolean withCache, boolean withLightingCache) { searchLineageCacheConfiguration.setTtlSeconds(600L); searchLineageCacheConfiguration.setLightningThreshold(withLightingCache ? -1 : 300); - _lineageSearchService = new LineageSearchService( + _lineageSearchService = spy(new LineageSearchService( new SearchService( new EntityDocCountCache(_entityRegistry, _elasticSearchService, entityDocCountCacheConfiguration), cachingEntitySearchService, @@ -143,7 +143,7 @@ private void resetService(boolean withCache, boolean withLightingCache) { 100, true), new SimpleRanker()), - _graphService, _cacheManager.getCache("test"), withCache, searchLineageCacheConfiguration); + _graphService, _cacheManager.getCache("test"), withCache, searchLineageCacheConfiguration)); } @BeforeMethod @@ -371,16 +371,19 @@ public void testLightningSearchService() throws Exception { assertEquals(searchResult.getNumEntities().intValue(), 1); assertEquals(searchResult.getEntities().get(0).getEntity(), urn); assertEquals(searchResult.getEntities().get(0).getDegree().intValue(), 1); + verify(_lineageSearchService, times(1)).getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); searchResult = searchAcrossLineage(QueryUtils.newFilter("degree.keyword", "1"), testStar); assertEquals(searchResult.getNumEntities().intValue(), 1); assertEquals(searchResult.getEntities().get(0).getEntity(), urn); assertEquals(searchResult.getEntities().get(0).getDegree().intValue(), 1); + verify(_lineageSearchService, times(2)).getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); searchResult = searchAcrossLineage(QueryUtils.newFilter("degree.keyword", "2"), testStar); assertEquals(searchResult.getNumEntities().intValue(), 0); assertEquals(searchResult.getEntities().size(), 0); - clearCache(true); + verify(_lineageSearchService, times(3)).getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); + clearCache(true); // resets spy Urn urn2 = new TestEntityUrn("test2", "urn2", "VALUE_2"); ObjectNode document2 = JsonNodeFactory.instance.objectNode(); @@ -394,6 +397,7 @@ public void testLightningSearchService() throws Exception { searchResult = searchAcrossLineage(null, testStar); assertEquals(searchResult.getNumEntities().intValue(), 1); assertEquals(searchResult.getEntities().get(0).getEntity(), urn); + verify(_lineageSearchService, times(1)).getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); clearCache(true); when(_graphService.getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), @@ -402,10 +406,12 @@ public void testLightningSearchService() throws Exception { searchResult = searchAcrossLineage(null, testStar); assertEquals(searchResult.getNumEntities().intValue(), 1); assertEquals(searchResult.getEntities().size(), 1); + verify(_lineageSearchService, times(1)).getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); clearCache(true); // Test Cache Behavior - Mockito.reset(_graphService); + reset(_graphService); + reset(_lineageSearchService); // Case 1: Use the maxHops in the cache. when(_graphService.getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), @@ -421,16 +427,18 @@ public void testLightningSearchService() throws Exception { new SearchFlags().setSkipCache(false)); assertEquals(searchResult.getNumEntities().intValue(), 1); - Mockito.verify(_graphService, times(1)).getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), + verify(_graphService, times(1)).getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), eq(1000), eq(null), eq(null)); + verify(_lineageSearchService, times(1)).getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); // Hit the cache on second attempt searchResult = _lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME), "*", 1000, null, null, 0, 10, null, null, new SearchFlags().setSkipCache(false)); assertEquals(searchResult.getNumEntities().intValue(), 1); - Mockito.verify(_graphService, times(1)).getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), + verify(_graphService, times(1)).getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), eq(1000), eq(null), eq(null)); + verify(_lineageSearchService, times(2)).getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); // Case 2: Use the start and end time in the cache. @@ -447,20 +455,24 @@ public void testLightningSearchService() throws Exception { new SearchFlags().setSkipCache(false)); assertEquals(searchResult.getNumEntities().intValue(), 1); - Mockito.verify(_graphService, times(1)).getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), + verify(_graphService, times(1)).getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), eq(1000), eq(0L), eq(1L)); + verify(_lineageSearchService, times(3)).getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); // Hit the cache on second attempt searchResult = _lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME), "*", null, null, null, 0, 10, 0L, 1L, new SearchFlags().setSkipCache(false)); assertEquals(searchResult.getNumEntities().intValue(), 1); - Mockito.verify(_graphService, times(1)).getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), + verify(_graphService, times(1)).getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), eq(1000), eq(0L), eq(1L)); + verify(_lineageSearchService, times(4)).getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); /* * Test filtering */ + reset(_lineageSearchService); + // Entity searchResult = _lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(DATASET_ENTITY_NAME), @@ -468,6 +480,7 @@ public void testLightningSearchService() throws Exception { new SearchFlags().setSkipCache(false)); assertEquals(searchResult.getNumEntities().intValue(), 0); assertEquals(searchResult.getEntities().size(), 0); + verify(_lineageSearchService, times(1)).getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); // Cached searchResult = @@ -476,25 +489,35 @@ public void testLightningSearchService() throws Exception { new SearchFlags().setSkipCache(false)); Mockito.verify(_graphService, times(1)).getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), eq(1000), eq(0L), eq(1L)); + verify(_lineageSearchService, times(2)).getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); assertEquals(searchResult.getNumEntities().intValue(), 0); assertEquals(searchResult.getEntities().size(), 0); // Platform - Filter filter = QueryUtils.newFilter("platform", "urn:li:dataPlatform:kafka"); + ConjunctiveCriterionArray conCritArr = new ConjunctiveCriterionArray(); + Criterion platform1Crit = new Criterion().setField("platform").setValue("urn:li:dataPlatform:kafka").setCondition(Condition.EQUAL); + CriterionArray critArr = new CriterionArray(ImmutableList.of(platform1Crit)); + conCritArr.add(new ConjunctiveCriterion().setAnd(critArr)); + Criterion degreeCrit = new Criterion().setField("degree.keyword").setValue("2").setCondition(Condition.EQUAL); + conCritArr.add(new ConjunctiveCriterion().setAnd(new CriterionArray(ImmutableList.of(degreeCrit)))); + Filter filter = new Filter().setOr(conCritArr); + searchResult = _lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME), "*", 1000, filter, null, 0, 10, null, null, new SearchFlags().setSkipCache(false)); assertEquals(searchResult.getNumEntities().intValue(), 0); assertEquals(searchResult.getEntities().size(), 0); + verify(_lineageSearchService, times(3)).getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); // Cached searchResult = _lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME), "*", 1000, filter, null, 0, 10, null, null, new SearchFlags().setSkipCache(false)); - Mockito.verify(_graphService, times(1)).getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), + verify(_graphService, times(1)).getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), eq(1000), eq(0L), eq(1L)); + verify(_lineageSearchService, times(4)).getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); assertEquals(searchResult.getNumEntities().intValue(), 0); assertEquals(searchResult.getEntities().size(), 0); @@ -506,14 +529,16 @@ public void testLightningSearchService() throws Exception { new SearchFlags().setSkipCache(false)); assertEquals(searchResult.getNumEntities().intValue(), 0); assertEquals(searchResult.getEntities().size(), 0); + verify(_lineageSearchService, times(5)).getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); // Cached searchResult = _lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME), "*", 1000, originFilter, null, 0, 10, null, null, new SearchFlags().setSkipCache(false)); - Mockito.verify(_graphService, times(1)).getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), + verify(_graphService, times(1)).getLineage(eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), eq(1000), eq(0L), eq(1L)); + verify(_lineageSearchService, times(6)).getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); assertEquals(searchResult.getNumEntities().intValue(), 0); assertEquals(searchResult.getEntities().size(), 0); @@ -749,7 +774,5 @@ public void testCanDoLightning() throws Exception { size = 10; filter = new Filter().setOr(conCritArr); Assert.assertTrue(_lineageSearchService.canDoLightning(lineageRelationships, "*", filter, null)); - - } } From 4b32e9a9abef88cf47595770367ab571631c8633 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Fri, 28 Jul 2023 21:07:03 -0500 Subject: [PATCH 196/222] docs(docker): document docker container dependency tree (#8496) --- docs-website/sidebars.js | 1 + docs/architecture/docker-containers.md | 27 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 docs/architecture/docker-containers.md diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index 7d651fd5d1894..2f584e2bd2e11 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -445,6 +445,7 @@ module.exports = { "docs/components", "docs/architecture/metadata-ingestion", "docs/architecture/metadata-serving", + "docs/architecture/docker-containers", ], }, { diff --git a/docs/architecture/docker-containers.md b/docs/architecture/docker-containers.md new file mode 100644 index 0000000000000..099a4fe21597a --- /dev/null +++ b/docs/architecture/docker-containers.md @@ -0,0 +1,27 @@ +--- +title: "Docker Container Architecture" +--- + +# Docker Container Architecture + +When running DataHub via docker-compose. or helm, the following is a diagram of the containers involved +with running DataHub and their relationships with each other. The helm chart uses helm hooks to determine +the proper ordering of the components whereas docker-compose relies on a series of health checks. + +```text + datahub-frontend-react datahub-actions + \ / + | datahub-upgrade (NoCodeDataMigration, helm only) + | / + datahub-gms (healthy) + | + datahub-upgrade (SystemUpdate completed) + /--------------------/ | \ \------------------------------------------------\ + / | \-------------------\ \ +mysql-setup (completed) elasticsearch-setup (completed) kafka-setup (completed) (if apply) neo4j (healthy) + | | / \ + | | / \ +mysql (healthy) elasticsearch (healthy) broker (healthy) (if not internal) schema-registry (healthy) + | + zookeeper (healthy) +``` \ No newline at end of file From e09a2e6c85460f6842b7a066963496cdf756522a Mon Sep 17 00:00:00 2001 From: Indy Prentice Date: Sat, 29 Jul 2023 04:56:49 -0300 Subject: [PATCH 197/222] feat(lineage): Apply search flags to scroll query in LineageSearchService (#8518) Co-authored-by: Indy Prentice --- .../com/linkedin/metadata/search/LineageSearchService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java index 930a55c56cb6e..c561ddd38b919 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java @@ -606,6 +606,7 @@ public LineageScrollResult scrollAcrossLineage(@Nonnull Urn sourceUrn, @Nonnull private LineageScrollResult getScrollResultInBatches(List lineageRelationships, @Nonnull String input, @Nullable Filter inputFilters, @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nonnull String keepAlive, int size, @Nonnull SearchFlags searchFlags) { + final SearchFlags finalFlags = applyDefaultSearchFlags(searchFlags, input, DEFAULT_SERVICE_SEARCH_FLAGS); LineageScrollResult finalResult = new LineageScrollResult().setEntities(new LineageSearchEntityArray(Collections.emptyList())) .setMetadata(new SearchResultMetadata().setAggregations(new AggregationMetadataArray())) @@ -623,7 +624,7 @@ private LineageScrollResult getScrollResultInBatches(List l LineageScrollResult resultForBatch = buildLineageScrollResult( _searchService.scrollAcrossEntities(entitiesToQuery, input, finalFilter, sortCriterion, scrollId, keepAlive, querySize, - searchFlags), urnToRelationship); + finalFlags), urnToRelationship); querySize = Math.max(0, size - resultForBatch.getEntities().size()); finalResult = mergeScrollResult(finalResult, resultForBatch); } From 55c6592575fdfa314601b1020b8e03f564321c6b Mon Sep 17 00:00:00 2001 From: Indy Prentice Date: Sat, 29 Jul 2023 04:57:10 -0300 Subject: [PATCH 198/222] feat(search): Throw exception instead of returning an empty response from scroll in an error case (#8517) Co-authored-by: Indy Prentice --- .../search/elasticsearch/query/ESSearchDAO.java | 10 ---------- .../metadata/search/utils/SearchUtils.java | 16 ---------------- 2 files changed, 26 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java index 57e8967c83985..e204cb6fd6fbe 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java @@ -33,7 +33,6 @@ import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Request; @@ -44,7 +43,6 @@ import static com.linkedin.metadata.Constants.*; import static com.linkedin.metadata.models.registry.template.util.TemplateUtil.*; -import static com.linkedin.metadata.search.utils.SearchUtils.*; import static com.linkedin.metadata.utils.SearchUtil.*; @@ -154,14 +152,6 @@ private ScrollResult executeAndExtract(@Nonnull List entitySpecs, @N .extractScrollResult(searchResponse, filter, scrollId, keepAlive, size, supportsPointInTime())); } catch (Exception e) { - if (e instanceof ElasticsearchStatusException) { - final ElasticsearchStatusException statusException = (ElasticsearchStatusException) e; - if (statusException.status().getStatus() == 400) { - // Malformed query -- Could indicate bad search syntax. Return empty response. - log.warn("Received 400 from Elasticsearch. Returning empty search response", e); - return EMPTY_SCROLL_RESULT; - } - } log.error("Search query failed", e); throw new ESQueryException("Search query failed:", e); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/SearchUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/SearchUtils.java index 5b9ce7444ae38..35a322d37b2fd 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/SearchUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/SearchUtils.java @@ -12,11 +12,8 @@ import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.search.AggregationMetadata; import com.linkedin.metadata.search.FilterValueArray; -import com.linkedin.metadata.search.ScrollResult; import com.linkedin.metadata.search.SearchEntity; -import com.linkedin.metadata.search.SearchEntityArray; import com.linkedin.metadata.search.SearchResult; -import com.linkedin.metadata.search.SearchResultMetadata; import com.linkedin.metadata.utils.SearchUtil; import java.io.IOException; import java.io.InputStream; @@ -39,19 +36,6 @@ @Slf4j public class SearchUtils { - public static final SearchResult EMPTY_SEARCH_RESULT = - new SearchResult().setEntities(new SearchEntityArray(Collections.emptyList())) - .setMetadata(new SearchResultMetadata()) - .setFrom(0) - .setPageSize(0) - .setNumEntities(0); - - public static final ScrollResult EMPTY_SCROLL_RESULT = - new ScrollResult().setEntities(new SearchEntityArray(Collections.emptyList())) - .setMetadata(new SearchResultMetadata()) - .setPageSize(0) - .setNumEntities(0); - private SearchUtils() { } From 753b0f1acc6994694d0c6cace0499b05ef352c0d Mon Sep 17 00:00:00 2001 From: Yang Jiandan Date: Sat, 29 Jul 2023 19:28:11 +0800 Subject: [PATCH 199/222] fix(gms): GMS hang when upgrade image #8270 (#8271) Co-authored-by: yangjd33 Co-authored-by: david-leifker <114954101+david-leifker@users.noreply.github.com> --- .../metadata/boot/kafka/DataHubUpgradeKafkaListener.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/DataHubUpgradeKafkaListener.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/DataHubUpgradeKafkaListener.java index e3f3a9c5b76a5..11d12072e12b7 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/DataHubUpgradeKafkaListener.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/DataHubUpgradeKafkaListener.java @@ -108,6 +108,10 @@ public void checkSystemVersion(final ConsumerRecord consu } public void waitForUpdate() { + if (!_configurationProvider.getSystemUpdate().isWaitForSystemUpdate()) { + log.warn("Wait for system update is disabled. Proceeding with startup."); + IS_UPDATED.getAndSet(true); + } int maxBackOffs = Integer.parseInt(_configurationProvider.getSystemUpdate().getMaxBackOffs()); long initialBackOffMs = Long.parseLong(_configurationProvider.getSystemUpdate().getInitialBackOffMs()); int backOffFactor = Integer.parseInt(_configurationProvider.getSystemUpdate().getBackOffFactor()); From dafb15071a230cbb9eff2e1565526606e6bc1c9b Mon Sep 17 00:00:00 2001 From: Sukeerthi Mandyam <69302927+Sukeerthi31@users.noreply.github.com> Date: Sun, 30 Jul 2023 15:55:32 +0530 Subject: [PATCH 200/222] fix(ui): Allows deselection of members in add members modal for a group (#8349) --- .../src/app/entity/group/AddGroupMembersModal.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datahub-web-react/src/app/entity/group/AddGroupMembersModal.tsx b/datahub-web-react/src/app/entity/group/AddGroupMembersModal.tsx index 25637fe91137a..45287e823523b 100644 --- a/datahub-web-react/src/app/entity/group/AddGroupMembersModal.tsx +++ b/datahub-web-react/src/app/entity/group/AddGroupMembersModal.tsx @@ -83,9 +83,9 @@ export const AddGroupMembersModal = ({ urn, visible, onCloseModal, onSubmit }: P setSelectedMembers(newUsers); }; - const onDeselectMember = (memberUrn: string) => { + const onDeselectMember = (memberUrn: { key: string; label: React.ReactNode; value: string }) => { setInputValue(''); - const newUserActors = selectedMembers.filter((user) => user !== memberUrn); + const newUserActors = selectedMembers.filter((user) => user.value !== memberUrn.value); setSelectedMembers(newUserActors); }; From 564d8a3760f2deaee6db919b5424f3d3d3aec090 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Mon, 31 Jul 2023 00:27:25 -0400 Subject: [PATCH 201/222] fix(ui) Remove initial redirect logic from frontend (#8401) --- .../src/app/context/UserContextProvider.tsx | 6 --- .../src/app/context/useInitialRedirect.ts | 52 ------------------- .../src/app/context/userContext.tsx | 5 -- 3 files changed, 63 deletions(-) delete mode 100644 datahub-web-react/src/app/context/useInitialRedirect.ts diff --git a/datahub-web-react/src/app/context/UserContextProvider.tsx b/datahub-web-react/src/app/context/UserContextProvider.tsx index 4a9f24cdea30e..3bcff15cc2748 100644 --- a/datahub-web-react/src/app/context/UserContextProvider.tsx +++ b/datahub-web-react/src/app/context/UserContextProvider.tsx @@ -3,7 +3,6 @@ import { useGetMeLazyQuery } from '../../graphql/me.generated'; import { useGetGlobalViewsSettingsLazyQuery } from '../../graphql/app.generated'; import { CorpUser, PlatformPrivileges } from '../../types.generated'; import { UserContext, LocalState, DEFAULT_STATE, State } from './userContext'; -import { useInitialRedirect } from './useInitialRedirect'; // TODO: Migrate all usage of useAuthenticatedUser to using this provider. @@ -125,11 +124,6 @@ const UserContextProvider = ({ children }: { children: React.ReactNode }) => { } }, [state, localState.selectedViewUrn, setDefaultSelectedView]); - /** - * Route to the most recently visited path once on first load of home page, if present in local storage. - */ - useInitialRedirect(state, localState, setState, updateLocalState); - return ( { - if (!state.loadedInitialPath) { - if (location.pathname === PageRoutes.ROOT && localState.selectedPath !== location.pathname) { - if (localState.selectedPath && !localState.selectedPath.includes(PageRoutes.EMBED)) { - history.replace({ - pathname: localState.selectedPath, - search: localState.selectedSearch || '', - }); - } - } - setState({ - ...state, - loadedInitialPath: true, - }); - } - }, [ - localState.selectedPath, - localState.selectedSearch, - location.pathname, - location.search, - state, - history, - setState, - ]); - - /** - * When the location of the browse changes, save the latest to local state. - */ - useEffect(() => { - if ( - (localState.selectedPath !== location.pathname || localState.selectedSearch !== location.search) && - !location.pathname.includes(PageRoutes.EMBED) - ) { - setLocalState({ - ...localState, - selectedPath: location.pathname, - selectedSearch: location.search, - }); - } - }, [location.pathname, location.search, localState, setLocalState]); -} diff --git a/datahub-web-react/src/app/context/userContext.tsx b/datahub-web-react/src/app/context/userContext.tsx index b6cbe6250e4ed..2e174d38247f5 100644 --- a/datahub-web-react/src/app/context/userContext.tsx +++ b/datahub-web-react/src/app/context/userContext.tsx @@ -22,10 +22,6 @@ export type State = { loadedPersonalDefaultViewUrn: boolean; hasSetDefaultView: boolean; }; - /** - * Whether the initial page path has been loaded. - */ - loadedInitialPath: boolean; }; /** @@ -54,7 +50,6 @@ export const DEFAULT_STATE: State = { loadedPersonalDefaultViewUrn: false, hasSetDefaultView: false, }, - loadedInitialPath: false, }; export const DEFAULT_CONTEXT = { From 2ca7f566f8c57453d2289d0c430b0207213bd1b1 Mon Sep 17 00:00:00 2001 From: Meenakshi Kamalaseshan Radha <62914384+mkamalas@users.noreply.github.com> Date: Mon, 31 Jul 2023 09:57:49 +0530 Subject: [PATCH 202/222] fix(sso) - Add redirect_uri to authenticate route on 401 error (#8346) --- datahub-web-react/src/App.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datahub-web-react/src/App.tsx b/datahub-web-react/src/App.tsx index 1d9c9cbd27dbb..68a4b93d71481 100644 --- a/datahub-web-react/src/App.tsx +++ b/datahub-web-react/src/App.tsx @@ -48,7 +48,8 @@ const errorLink = onError((error) => { if (serverError.statusCode === 401) { isLoggedInVar(false); Cookies.remove(GlobalCfg.CLIENT_AUTH_COOKIE); - window.location.replace(PageRoutes.AUTHENTICATE); + const currentPath = window.location.pathname + window.location.search; + window.location.replace(`${PageRoutes.AUTHENTICATE}?redirect_uri=${encodeURIComponent(currentPath)}`); } } if (graphQLErrors && graphQLErrors.length) { From 352a54a7bf3ece31d4927ee807599471c94abcc2 Mon Sep 17 00:00:00 2001 From: Xuelei Li <115022112+lix-mms@users.noreply.github.com> Date: Mon, 31 Jul 2023 06:28:02 +0200 Subject: [PATCH 203/222] fix(auth): ignore case when comparing http headers (#8356) --- datahub-frontend/app/controllers/Application.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/datahub-frontend/app/controllers/Application.java b/datahub-frontend/app/controllers/Application.java index d31cb13fa7b41..5c76f2572a936 100644 --- a/datahub-frontend/app/controllers/Application.java +++ b/datahub-frontend/app/controllers/Application.java @@ -136,11 +136,11 @@ public CompletableFuture proxy(String path, Http.Request request) throws .stream() // Remove X-DataHub-Actor to prevent malicious delegation. .filter(entry -> !AuthenticationConstants.LEGACY_X_DATAHUB_ACTOR_HEADER.equalsIgnoreCase(entry.getKey())) - .filter(entry -> !Http.HeaderNames.CONTENT_LENGTH.equals(entry.getKey())) - .filter(entry -> !Http.HeaderNames.CONTENT_TYPE.equals(entry.getKey())) - .filter(entry -> !Http.HeaderNames.AUTHORIZATION.equals(entry.getKey())) + .filter(entry -> !Http.HeaderNames.CONTENT_LENGTH.equalsIgnoreCase(entry.getKey())) + .filter(entry -> !Http.HeaderNames.CONTENT_TYPE.equalsIgnoreCase(entry.getKey())) + .filter(entry -> !Http.HeaderNames.AUTHORIZATION.equalsIgnoreCase(entry.getKey())) // Remove Host s.th. service meshes do not route to wrong host - .filter(entry -> !Http.HeaderNames.HOST.equals(entry.getKey())) + .filter(entry -> !Http.HeaderNames.HOST.equalsIgnoreCase(entry.getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) ) .addHeader(Http.HeaderNames.AUTHORIZATION, authorizationHeaderValue) @@ -152,8 +152,8 @@ public CompletableFuture proxy(String path, Http.Request request) throws final ResponseHeader header = new ResponseHeader(apiResponse.getStatus(), apiResponse.getHeaders() .entrySet() .stream() - .filter(entry -> !Http.HeaderNames.CONTENT_LENGTH.equals(entry.getKey())) - .filter(entry -> !Http.HeaderNames.CONTENT_TYPE.equals(entry.getKey())) + .filter(entry -> !Http.HeaderNames.CONTENT_LENGTH.equalsIgnoreCase(entry.getKey())) + .filter(entry -> !Http.HeaderNames.CONTENT_TYPE.equalsIgnoreCase(entry.getKey())) .map(entry -> Pair.of(entry.getKey(), String.join(";", entry.getValue()))) .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond))); final HttpEntity body = new HttpEntity.Strict(apiResponse.getBodyAsBytes(), Optional.ofNullable(apiResponse.getContentType())); From 1e507bcd408fdc59548dab895370e4b7cb9c9890 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20L=C3=BCdin?= <13187726+Masterchen09@users.noreply.github.com> Date: Mon, 31 Jul 2023 06:28:19 +0200 Subject: [PATCH 204/222] fix(ui): use locale lowercase when filtering columns of an entity in the lineage (#8213) --- .../src/app/lineage/utils/columnLineageUtils.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/datahub-web-react/src/app/lineage/utils/columnLineageUtils.ts b/datahub-web-react/src/app/lineage/utils/columnLineageUtils.ts index 505b3d94531b7..4dd54ea25416d 100644 --- a/datahub-web-react/src/app/lineage/utils/columnLineageUtils.ts +++ b/datahub-web-react/src/app/lineage/utils/columnLineageUtils.ts @@ -138,7 +138,10 @@ export function filterColumns( node: { x: number; y: number; data: Omit }, setColumnsByUrn: (value: React.SetStateAction>) => void, ) { - const filteredFields = node.data.schemaMetadata?.fields.filter((field) => field.fieldPath.includes(filterText)); + const formattedFilterText = filterText.toLocaleLowerCase(); + const filteredFields = node.data.schemaMetadata?.fields.filter((field) => + field.fieldPath.toLocaleLowerCase().includes(formattedFilterText), + ); if (filteredFields) { setColumnsByUrn((colsByUrn) => ({ ...colsByUrn, From b77b4e2ba9a946709acf18af9fe03d6fdfc1ed62 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Sun, 30 Jul 2023 23:35:26 -0500 Subject: [PATCH 205/222] feat(elasticsearch): allow bulk delete (#8424) --- .../elasticsearch/update/ESBulkProcessor.java | 14 ++++++++++---- .../src/main/resources/application.yml | 1 + .../search/ElasticSearchBulkProcessorFactory.java | 4 ++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/update/ESBulkProcessor.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/update/ESBulkProcessor.java index b53cf80e4a59e..a7ece47a7f5d6 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/update/ESBulkProcessor.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/update/ESBulkProcessor.java @@ -47,6 +47,9 @@ public static ESBulkProcessor.ESBulkProcessorBuilder builder(RestHighLevelClient @NonNull private Boolean async = false; @Builder.Default + @NonNull + private Boolean batchDelete = false; + @Builder.Default private Integer bulkRequestsLimit = 500; @Builder.Default private Integer bulkFlushPeriod = 1; @@ -62,12 +65,13 @@ public static ESBulkProcessor.ESBulkProcessorBuilder builder(RestHighLevelClient @Getter(AccessLevel.NONE) private final BulkProcessor bulkProcessor; - private ESBulkProcessor(@NonNull RestHighLevelClient searchClient, @NonNull Boolean async, Integer bulkRequestsLimit, - Integer bulkFlushPeriod, Integer numRetries, Long retryInterval, + private ESBulkProcessor(@NonNull RestHighLevelClient searchClient, @NonNull Boolean async, @NonNull Boolean batchDelete, + Integer bulkRequestsLimit, Integer bulkFlushPeriod, Integer numRetries, Long retryInterval, TimeValue defaultTimeout, WriteRequest.RefreshPolicy writeRequestRefreshPolicy, BulkProcessor ignored) { this.searchClient = searchClient; this.async = async; + this.batchDelete = batchDelete; this.bulkRequestsLimit = bulkRequestsLimit; this.bulkFlushPeriod = bulkFlushPeriod; this.numRetries = numRetries; @@ -103,8 +107,10 @@ public Optional deleteByQuery(QueryBuilder queryBuilder, b deleteByQueryRequest.indices(indices); try { - // flush pending writes - bulkProcessor.flush(); + if (!batchDelete) { + // flush pending writes + bulkProcessor.flush(); + } // perform delete after local flush final BulkByScrollResponse deleteResponse = searchClient.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); MetricUtils.counter(this.getClass(), ES_WRITES_METRIC).inc(deleteResponse.getTotal()); diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml index 18f042b65d0b1..5290dd4adceac 100644 --- a/metadata-service/configuration/src/main/resources/application.yml +++ b/metadata-service/configuration/src/main/resources/application.yml @@ -168,6 +168,7 @@ elasticsearch: numRetries: ${ES_BULK_NUM_RETRIES:3} retryInterval: ${ES_BULK_RETRY_INTERVAL:1} refreshPolicy: ${ES_BULK_REFRESH_POLICY:NONE} + enableBatchDelete: ${ES_BULK_ENABLE_BATCH_DELETE:false} index: prefix: ${INDEX_PREFIX:} numShards: ${ELASTICSEARCH_NUM_SHARDS_PER_INDEX:1} diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchBulkProcessorFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchBulkProcessorFactory.java index 60bb89cf3c589..956157f70e6bc 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchBulkProcessorFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchBulkProcessorFactory.java @@ -41,6 +41,9 @@ public class ElasticSearchBulkProcessorFactory { @Value("#{new Boolean('${elasticsearch.bulkProcessor.async}')}") private boolean async; + @Value("#{new Boolean('${elasticsearch.bulkProcessor.enableBatchDelete}')}") + private boolean enableBatchDelete; + @Value("${elasticsearch.bulkProcessor.refreshPolicy}") private String refreshPolicy; @@ -53,6 +56,7 @@ protected ESBulkProcessor getInstance() { .bulkRequestsLimit(bulkRequestsLimit) .retryInterval(retryInterval) .numRetries(numRetries) + .batchDelete(enableBatchDelete) .writeRequestRefreshPolicy(WriteRequest.RefreshPolicy.valueOf(refreshPolicy)) .build(); } From de1f23d9d9fa391f24b3198d1ed6685e5d3700af Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Mon, 31 Jul 2023 09:10:40 -0500 Subject: [PATCH 206/222] feat(metrics): add metrics for aspect write and bytes (#8526) --- .../com/linkedin/metadata/entity/AspectDao.java | 10 ++++++++++ .../metadata/entity/EntityServiceImpl.java | 17 +++++++++++++++-- .../entity/cassandra/CassandraAspectDao.java | 4 ++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectDao.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectDao.java index bf74b1025267f..9967df9207ec7 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectDao.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectDao.java @@ -3,6 +3,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.metadata.entity.ebean.EbeanAspectV2; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; +import com.linkedin.metadata.utils.metrics.MetricUtils; import io.ebean.PagedList; import javax.annotation.Nonnull; @@ -28,6 +29,8 @@ * worth looking into ways to move this responsibility inside {@link AspectDao} implementations. */ public interface AspectDao { + String ASPECT_WRITE_COUNT_METRIC_NAME = "aspectWriteCount"; + String ASPECT_WRITE_BYTES_METRIC_NAME = "aspectWriteBytes"; @Nullable EntityAspect getAspect(@Nonnull final String urn, @Nonnull final String aspectName, final long version); @@ -116,4 +119,11 @@ ListResult listAspectMetadata( @Nonnull T runInTransactionWithRetry(@Nonnull final Supplier block, final int maxTransactionRetry); + + default void incrementWriteMetrics(String aspectName, long count, long bytes) { + MetricUtils.counter(this.getClass(), + String.join(MetricUtils.DELIMITER, List.of(ASPECT_WRITE_COUNT_METRIC_NAME, aspectName))).inc(count); + MetricUtils.counter(this.getClass(), + String.join(MetricUtils.DELIMITER, List.of(ASPECT_WRITE_BYTES_METRIC_NAME, aspectName))).inc(bytes); + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java index c7a9895992d90..e070944b49a05 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java @@ -1811,6 +1811,8 @@ public RollbackResult deleteAspect(String urn, String aspectName, @Nonnull Map Date: Mon, 31 Jul 2023 16:13:07 +0200 Subject: [PATCH 207/222] fix(ingest/build): Fix sagemaker mypy and flake8 issues (#8530) --- .../aws/sagemaker_processors/feature_groups.py | 4 ++-- metadata-ingestion/src/datahub/utilities/mapping.py | 8 ++++---- metadata-ingestion/src/datahub_provider/_plugin.py | 2 +- .../operators/datahub_assertion_operator.py | 4 ++-- .../operators/datahub_assertion_sensor.py | 4 ++-- .../operators/datahub_operation_operator.py | 4 ++-- .../operators/datahub_operation_sensor.py | 4 ++-- metadata-ingestion/tests/unit/test_glue_source.py | 2 +- metadata-ingestion/tests/unit/test_kafka_source.py | 12 ++++++++---- .../unit/utilities/test_file_backed_collections.py | 2 +- 10 files changed, 25 insertions(+), 21 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py index 75fae47c966fd..b8b96c6306a3b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py @@ -24,7 +24,7 @@ from mypy_boto3_sagemaker import SageMakerClient from mypy_boto3_sagemaker.type_defs import ( DescribeFeatureGroupResponseTypeDef, - FeatureDefinitionOutputTypeDef, + FeatureDefinitionTypeDef, FeatureGroupSummaryTypeDef, ) @@ -147,7 +147,7 @@ def get_feature_type(self, aws_type: str, feature_name: str) -> str: def get_feature_wu( self, feature_group_details: "DescribeFeatureGroupResponseTypeDef", - feature: "FeatureDefinitionOutputTypeDef", + feature: "FeatureDefinitionTypeDef", ) -> MetadataWorkUnit: """ Generate an MLFeature workunit for a SageMaker feature. diff --git a/metadata-ingestion/src/datahub/utilities/mapping.py b/metadata-ingestion/src/datahub/utilities/mapping.py index e7c60435e13f8..32666ceecdf85 100644 --- a/metadata-ingestion/src/datahub/utilities/mapping.py +++ b/metadata-ingestion/src/datahub/utilities/mapping.py @@ -252,11 +252,11 @@ def sanitize_owner_ids(self, owner_id: str) -> str: def get_match(self, match_clause: Any, raw_props_value: Any) -> Optional[Match]: # function to check if a match clause is satisfied to a value. - if type(raw_props_value) not in Constants.OPERAND_DATATYPE_SUPPORTED or type( - raw_props_value - ) != type(match_clause): + if not any( + isinstance(raw_props_value, t) for t in Constants.OPERAND_DATATYPE_SUPPORTED + ) or not isinstance(raw_props_value, type(match_clause)): return None - elif type(raw_props_value) == str: + elif isinstance(raw_props_value, str): return re.match(match_clause, raw_props_value) else: return re.match(str(match_clause), str(raw_props_value)) diff --git a/metadata-ingestion/src/datahub_provider/_plugin.py b/metadata-ingestion/src/datahub_provider/_plugin.py index b4ac5a36c6eae..6f6c7c9ab71b7 100644 --- a/metadata-ingestion/src/datahub_provider/_plugin.py +++ b/metadata-ingestion/src/datahub_provider/_plugin.py @@ -107,7 +107,7 @@ def get_inlets_from_task(task: BaseOperator, context: Any) -> Iterable[Any]: ] for inlet in task_inlets: - if type(inlet) != str: + if isinstance(inlet, str): inlets.append(inlet) return inlets diff --git a/metadata-ingestion/src/datahub_provider/operators/datahub_assertion_operator.py b/metadata-ingestion/src/datahub_provider/operators/datahub_assertion_operator.py index 89a037324e7cb..28be8ad860179 100644 --- a/metadata-ingestion/src/datahub_provider/operators/datahub_assertion_operator.py +++ b/metadata-ingestion/src/datahub_provider/operators/datahub_assertion_operator.py @@ -62,9 +62,9 @@ def execute(self, context: Any) -> bool: return True self.log.info(f"Checking if dataset {self.urn} is ready to be consumed") - if type(self.urn) == str: + if isinstance(self.urn, str): urns = [self.urn] - elif type(self.urn) == list: + elif isinstance(self.urn, list): urns = self.urn else: raise Exception(f"urn parameter has invalid type {type(self.urn)}") diff --git a/metadata-ingestion/src/datahub_provider/operators/datahub_assertion_sensor.py b/metadata-ingestion/src/datahub_provider/operators/datahub_assertion_sensor.py index 55a3492f9c8d6..ceb970dd8dc7f 100644 --- a/metadata-ingestion/src/datahub_provider/operators/datahub_assertion_sensor.py +++ b/metadata-ingestion/src/datahub_provider/operators/datahub_assertion_sensor.py @@ -61,9 +61,9 @@ def poke(self, context: Any) -> bool: return True self.log.info(f"Checking if dataset {self.urn} is ready to be consumed") - if type(self.urn) == str: + if isinstance(self.urn, str): urns = [self.urn] - elif type(self.urn) == list: + elif isinstance(self.urn, list): urns = self.urn else: raise Exception(f"urn parameter has invalid type {type(self.urn)}") diff --git a/metadata-ingestion/src/datahub_provider/operators/datahub_operation_operator.py b/metadata-ingestion/src/datahub_provider/operators/datahub_operation_operator.py index e5e45c2bf4694..6b2535994c101 100644 --- a/metadata-ingestion/src/datahub_provider/operators/datahub_operation_operator.py +++ b/metadata-ingestion/src/datahub_provider/operators/datahub_operation_operator.py @@ -76,9 +76,9 @@ def execute(self, context: Any) -> bool: return True self.log.info(f"Checking if dataset {self.urn} is ready to be consumed") - if type(self.urn) == str: + if isinstance(self.urn, str): urns = [self.urn] - elif type(self.urn) == list: + elif isinstance(self.urn, list): urns = self.urn else: raise Exception(f"urn parameter has invalid type {type(self.urn)}") diff --git a/metadata-ingestion/src/datahub_provider/operators/datahub_operation_sensor.py b/metadata-ingestion/src/datahub_provider/operators/datahub_operation_sensor.py index 31b387a7e65b0..8796215453500 100644 --- a/metadata-ingestion/src/datahub_provider/operators/datahub_operation_sensor.py +++ b/metadata-ingestion/src/datahub_provider/operators/datahub_operation_sensor.py @@ -78,9 +78,9 @@ def poke(self, context: Any) -> bool: return True self.log.info(f"Checking if dataset {self.urn} is ready to be consumed") - if type(self.urn) == str: + if isinstance(self.urn, str): urns = [self.urn] - elif type(self.urn) == list: + elif isinstance(self.urn, list): urns = self.urn else: raise Exception(f"urn parameter has invalid type {type(self.urn)}") diff --git a/metadata-ingestion/tests/unit/test_glue_source.py b/metadata-ingestion/tests/unit/test_glue_source.py index 23dc3b97e09b5..8fb840ee003c7 100644 --- a/metadata-ingestion/tests/unit/test_glue_source.py +++ b/metadata-ingestion/tests/unit/test_glue_source.py @@ -89,7 +89,7 @@ def test_column_type(hive_column_type: str, expected_type: Type) -> None: ) schema_fields = avro_schema_to_mce_fields(json.dumps(avro_schema)) actual_schema_field_type = schema_fields[0].type - assert type(actual_schema_field_type.type) == expected_type + assert isinstance(actual_schema_field_type.type, expected_type) @pytest.mark.parametrize( diff --git a/metadata-ingestion/tests/unit/test_kafka_source.py b/metadata-ingestion/tests/unit/test_kafka_source.py index 33496b868adb7..b48ebf12ee37a 100644 --- a/metadata-ingestion/tests/unit/test_kafka_source.py +++ b/metadata-ingestion/tests/unit/test_kafka_source.py @@ -132,7 +132,9 @@ def test_kafka_source_workunits_with_platform_instance(mock_kafka, mock_admin_cl # DataPlatform aspect should be present when platform_instance is configured data_platform_aspects = [ - asp for asp in proposed_snap.aspects if type(asp) == DataPlatformInstanceClass + asp + for asp in proposed_snap.aspects + if isinstance(asp, DataPlatformInstanceClass) ] assert len(data_platform_aspects) == 1 assert data_platform_aspects[0].instance == make_dataplatform_instance_urn( @@ -141,7 +143,7 @@ def test_kafka_source_workunits_with_platform_instance(mock_kafka, mock_admin_cl # The default browse path should include the platform_instance value browse_path_aspects = [ - asp for asp in proposed_snap.aspects if type(asp) == BrowsePathsClass + asp for asp in proposed_snap.aspects if isinstance(asp, BrowsePathsClass) ] assert len(browse_path_aspects) == 1 assert f"/prod/{PLATFORM}/{PLATFORM_INSTANCE}" in browse_path_aspects[0].paths @@ -177,13 +179,15 @@ def test_kafka_source_workunits_no_platform_instance(mock_kafka, mock_admin_clie # DataPlatform aspect should not be present when platform_instance is not configured data_platform_aspects = [ - asp for asp in proposed_snap.aspects if type(asp) == DataPlatformInstanceClass + asp + for asp in proposed_snap.aspects + if isinstance(asp, DataPlatformInstanceClass) ] assert len(data_platform_aspects) == 0 # The default browse path should include the platform_instance value browse_path_aspects = [ - asp for asp in proposed_snap.aspects if type(asp) == BrowsePathsClass + asp for asp in proposed_snap.aspects if isinstance(asp, BrowsePathsClass) ] assert len(browse_path_aspects) == 1 assert f"/prod/{PLATFORM}" in browse_path_aspects[0].paths diff --git a/metadata-ingestion/tests/unit/utilities/test_file_backed_collections.py b/metadata-ingestion/tests/unit/utilities/test_file_backed_collections.py index 582e3814c4f6d..2d7556eb88341 100644 --- a/metadata-ingestion/tests/unit/utilities/test_file_backed_collections.py +++ b/metadata-ingestion/tests/unit/utilities/test_file_backed_collections.py @@ -256,7 +256,7 @@ def test_shared_connection() -> None: iterator = cache2.sql_query_iterator( f"SELECT y, sum(x) FROM {cache2.tablename} GROUP BY y ORDER BY y" ) - assert type(iterator) == sqlite3.Cursor + assert isinstance(iterator, sqlite3.Cursor) assert [tuple(r) for r in iterator] == [("a", 15), ("b", 11)] # Test joining between the two tables. From f884eb8791e57fa3cfd02accf3195ada63b97bd8 Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Mon, 31 Jul 2023 16:34:47 -0700 Subject: [PATCH 208/222] feat(siblings): hiding non-existant siblings in FE (#8528) --- .../shared/__tests__/siblingsUtils.test.ts | 178 ++++++++++++++++++ .../sidebar/SidebarSiblingsSection.tsx | 14 +- .../src/app/entity/shared/siblingUtils.ts | 7 +- .../src/app/search/SearchResultList.tsx | 4 +- datahub-web-react/src/graphql/dataset.graphql | 2 +- datahub-web-react/src/graphql/search.graphql | 2 + 6 files changed, 201 insertions(+), 6 deletions(-) diff --git a/datahub-web-react/src/app/entity/shared/__tests__/siblingsUtils.test.ts b/datahub-web-react/src/app/entity/shared/__tests__/siblingsUtils.test.ts index a15c4bc20ef29..6e23d5400ab77 100644 --- a/datahub-web-react/src/app/entity/shared/__tests__/siblingsUtils.test.ts +++ b/datahub-web-react/src/app/entity/shared/__tests__/siblingsUtils.test.ts @@ -195,6 +195,7 @@ const searchResultWithSiblings = [ { entity: { urn: 'urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)', + exists: true, type: 'DATASET', name: 'cypress_project.jaffle_shop.raw_orders', origin: 'PROD', @@ -328,6 +329,7 @@ const searchResultWithSiblings = [ siblings: [ { urn: 'urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)', + exists: true, type: 'DATASET', platform: { urn: 'urn:li:dataPlatform:dbt', @@ -376,6 +378,7 @@ const searchResultWithSiblings = [ { entity: { urn: 'urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)', + exists: true, type: 'DATASET', name: 'cypress_project.jaffle_shop.raw_orders', origin: 'PROD', @@ -513,6 +516,169 @@ const searchResultWithSiblings = [ }, ]; +const searchResultWithGhostSiblings = [ + { + entity: { + urn: 'urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)', + exists: true, + type: 'DATASET', + name: 'cypress_project.jaffle_shop.raw_orders', + origin: 'PROD', + uri: null, + platform: { + urn: 'urn:li:dataPlatform:bigquery', + type: 'DATA_PLATFORM', + name: 'bigquery', + properties: { + type: 'RELATIONAL_DB', + displayName: 'BigQuery', + datasetNameDelimiter: '.', + logoUrl: '/assets/platforms/bigquerylogo.png', + __typename: 'DataPlatformProperties', + }, + displayName: null, + info: null, + __typename: 'DataPlatform', + }, + dataPlatformInstance: null, + editableProperties: null, + platformNativeType: null, + properties: { + name: 'raw_orders', + description: null, + qualifiedName: null, + customProperties: [], + __typename: 'DatasetProperties', + }, + ownership: null, + globalTags: null, + glossaryTerms: null, + subTypes: { + typeNames: ['table'], + __typename: 'SubTypes', + }, + domain: null, + container: { + urn: 'urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb', + platform: { + urn: 'urn:li:dataPlatform:bigquery', + type: 'DATA_PLATFORM', + name: 'bigquery', + properties: { + type: 'RELATIONAL_DB', + displayName: 'BigQuery', + datasetNameDelimiter: '.', + logoUrl: '/assets/platforms/bigquerylogo.png', + __typename: 'DataPlatformProperties', + }, + displayName: null, + info: null, + __typename: 'DataPlatform', + }, + properties: { + name: 'jaffle_shop', + __typename: 'ContainerProperties', + }, + subTypes: { + typeNames: ['Dataset'], + __typename: 'SubTypes', + }, + deprecation: null, + __typename: 'Container', + }, + parentContainers: { + count: 2, + containers: [ + { + urn: 'urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb', + platform: { + urn: 'urn:li:dataPlatform:bigquery', + type: 'DATA_PLATFORM', + name: 'bigquery', + properties: { + type: 'RELATIONAL_DB', + displayName: 'BigQuery', + datasetNameDelimiter: '.', + logoUrl: '/assets/platforms/bigquerylogo.png', + __typename: 'DataPlatformProperties', + }, + displayName: null, + info: null, + __typename: 'DataPlatform', + }, + properties: { + name: 'jaffle_shop', + __typename: 'ContainerProperties', + }, + subTypes: { + typeNames: ['Dataset'], + __typename: 'SubTypes', + }, + deprecation: null, + __typename: 'Container', + }, + { + urn: 'urn:li:container:b5e95fce839e7d78151ed7e0a7420d84', + platform: { + urn: 'urn:li:dataPlatform:bigquery', + type: 'DATA_PLATFORM', + name: 'bigquery', + properties: { + type: 'RELATIONAL_DB', + displayName: 'BigQuery', + datasetNameDelimiter: '.', + logoUrl: '/assets/platforms/bigquerylogo.png', + __typename: 'DataPlatformProperties', + }, + displayName: null, + info: null, + __typename: 'DataPlatform', + }, + properties: { + name: 'cypress_project', + __typename: 'ContainerProperties', + }, + subTypes: { + typeNames: ['Project'], + __typename: 'SubTypes', + }, + deprecation: null, + __typename: 'Container', + }, + ], + __typename: 'ParentContainersResult', + }, + deprecation: null, + siblings: { + isPrimary: false, + siblings: [ + { + urn: 'urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)', + exists: false, + type: 'DATASET', + }, + ], + __typename: 'SiblingProperties', + }, + __typename: 'Dataset', + }, + matchedFields: [ + { + name: 'name', + value: 'raw_orders', + __typename: 'MatchedField', + }, + { + name: 'id', + value: 'cypress_project.jaffle_shop.raw_orders', + __typename: 'MatchedField', + }, + ], + insights: [], + __typename: 'SearchResult', + }, +]; + describe('siblingUtils', () => { describe('combineEntityDataWithSiblings', () => { it('combines my metadata with my siblings as primary', () => { @@ -564,6 +730,18 @@ describe('siblingUtils', () => { expect(result?.[0]?.matchedEntities?.[1]?.urn).toEqual( 'urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)', ); + + expect(result?.[0]?.matchedEntities).toHaveLength(2); + }); + + it('will not combine an entity with a ghost node', () => { + const result = combineSiblingsInSearchResults(searchResultWithGhostSiblings as any); + + expect(result).toHaveLength(1); + expect(result?.[0]?.matchedEntities?.[0]?.urn).toEqual( + 'urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)', + ); + expect(result?.[0]?.matchedEntities).toHaveLength(1); }); }); diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarSiblingsSection.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarSiblingsSection.tsx index 4b0089e6b9214..4ea1ab69e44b3 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarSiblingsSection.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarSiblingsSection.tsx @@ -4,7 +4,7 @@ import styled from 'styled-components'; import { useDataNotCombinedWithSiblings, useEntityData } from '../../../EntityContext'; import { SidebarHeader } from './SidebarHeader'; import { CompactEntityNameList } from '../../../../../recommendations/renderer/component/CompactEntityNameList'; -import { Entity } from '../../../../../../types.generated'; +import { Dataset, Entity } from '../../../../../../types.generated'; import { SEPARATE_SIBLINGS_URL_PARAM, stripSiblingsFromEntity, useIsSeparateSiblingsMode } from '../../../siblingUtils'; import { GetDatasetQuery } from '../../../../../../graphql/dataset.generated'; @@ -36,14 +36,22 @@ export const SidebarSiblingsSection = () => { const siblingEntities = entityData?.siblings?.siblings || []; const entityDataWithoutSiblings = stripSiblingsFromEntity(dataNotCombinedWithSiblings.dataset); - const allSiblingsInGroup = [...siblingEntities, entityDataWithoutSiblings] as Entity[]; + const allSiblingsInGroup = [...siblingEntities, entityDataWithoutSiblings] as Dataset[]; + + const allSiblingsInGroupThatExist = allSiblingsInGroup.filter((sibling) => sibling.exists); + + // you are always going to be in the sibling group, so if the sibling group is just you do not render. + // The less than case is likely not neccessary but just there as a safety case for unexpected scenarios + if (allSiblingsInGroupThatExist.length <= 1) { + return <>; + } return (
diff --git a/datahub-web-react/src/app/entity/shared/siblingUtils.ts b/datahub-web-react/src/app/entity/shared/siblingUtils.ts index 47687e4050b9b..2cad28d754a80 100644 --- a/datahub-web-react/src/app/entity/shared/siblingUtils.ts +++ b/datahub-web-react/src/app/entity/shared/siblingUtils.ts @@ -2,7 +2,7 @@ import merge from 'deepmerge'; import { unionBy, keyBy, values } from 'lodash'; import { useLocation } from 'react-router-dom'; import * as QueryString from 'query-string'; -import { Entity, MatchedField, Maybe, SiblingProperties } from '../../../types.generated'; +import { Dataset, Entity, MatchedField, Maybe, SiblingProperties } from '../../../types.generated'; export function stripSiblingsFromEntity(entity: any) { return { @@ -235,6 +235,11 @@ export function combineSiblingsInSearchResults( combinedResult.matchedEntities = entity.siblings.isPrimary ? [stripSiblingsFromEntity(entity), ...entity.siblings.siblings] : [...entity.siblings.siblings, stripSiblingsFromEntity(entity)]; + + combinedResult.matchedEntities = combinedResult.matchedEntities.filter( + (resultToFilter) => (resultToFilter as Dataset).exists, + ); + siblingUrns.forEach((urn) => { siblingsToPair[urn] = combinedResult; }); diff --git a/datahub-web-react/src/app/search/SearchResultList.tsx b/datahub-web-react/src/app/search/SearchResultList.tsx index a793348db6f1d..b860e7b670c33 100644 --- a/datahub-web-react/src/app/search/SearchResultList.tsx +++ b/datahub-web-react/src/app/search/SearchResultList.tsx @@ -151,7 +151,9 @@ export const SearchResultList = ({ )} {entityRegistry.renderSearchResult(item.entity.type, item)} - {item.matchedEntities && item.matchedEntities.length > 0 && ( + {/* an entity is always going to be inserted in the sibling group, so if the sibling group is just one do not + render. */} + {item.matchedEntities && item.matchedEntities.length > 1 && ( Date: Mon, 31 Jul 2023 19:48:05 -0700 Subject: [PATCH 209/222] fix(ingest): pin boto3-stubs in CI (#8527) Co-authored-by: Tamas Nemeth --- metadata-ingestion/setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 5498c96d91af6..d6738f4b9782c 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -401,7 +401,9 @@ def get_long_description(): "types-cachetools", # versions 0.1.13 and 0.1.14 seem to have issues "types-click==0.1.12", - "boto3-stubs[s3,glue,sagemaker,sts]>=1.28.4", + # The boto3-stubs package seems to have regularly breaking minor releases, + # we pin to a specific version to avoid this. + "boto3-stubs[s3,glue,sagemaker,sts]==1.28.15", "types-tabulate", # avrogen package requires this "types-pytz", From 0593e2355868e3c16afd89eef246cae357c07e75 Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Tue, 1 Aug 2023 01:00:51 -0700 Subject: [PATCH 210/222] docs: small update to homepage (#8483) Co-authored-by: Chris Collins --- docs-website/src/pages/_components/Hero/index.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs-website/src/pages/_components/Hero/index.js b/docs-website/src/pages/_components/Hero/index.js index ad6aa5f5dadba..b5fa04c80faee 100644 --- a/docs-website/src/pages/_components/Hero/index.js +++ b/docs-website/src/pages/_components/Hero/index.js @@ -27,11 +27,14 @@ const Hero = ({}) => { {/* HeroAnnouncement goes here */}
-

The #1 Open Source Data Catalog

+

The #1 Open Source Metadata Platform

- DataHub's extensible metadata platform enables data discovery, data observability and federated governance that helps tame the + DataHub is an extensible metadata platform that enables data discovery, data observability and federated governance to help tame the complexity of your data ecosystem.

+

+Built with ❤️ by Acryl Data and LinkedIn. +

Get Started → From 09ef6e080ce89b99ee6489d7ff56f311dce911ce Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Tue, 1 Aug 2023 19:31:26 +0530 Subject: [PATCH 211/222] fix(ingest): remove duplication of tags (#8532) --- .../datahub/ingestion/transformer/dataset_transformer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/dataset_transformer.py b/metadata-ingestion/src/datahub/ingestion/transformer/dataset_transformer.py index 0753d6c3fd830..0b2433c3a1fe2 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/dataset_transformer.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/dataset_transformer.py @@ -75,7 +75,11 @@ def update_if_keep_existing( ) -> None: """Check if user want to keep existing tags""" if in_global_tags_aspect is not None and config.replace_existing is False: - out_global_tags_aspect.tags.extend(in_global_tags_aspect.tags) + tags_seen = set() + for item in in_global_tags_aspect.tags: + if item.tag not in tags_seen: + out_global_tags_aspect.tags.append(item) + tags_seen.add(item.tag) @staticmethod def get_result_semantics( From a9bc5643f3ad20eea4ac31a98328d5c02f36b1a7 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 1 Aug 2023 07:04:11 -0700 Subject: [PATCH 212/222] ci: reduce git fetch depth (#8473) --- .github/workflows/build-and-test.yml | 2 +- .github/workflows/check-datahub-jars.yml | 2 +- .github/workflows/code-checks.yml | 2 +- .github/workflows/docker-ingestion-base.yml | 2 +- .github/workflows/docker-ingestion-smoke.yml | 2 +- .github/workflows/docker-ingestion.yml | 2 +- .github/workflows/docker-postgres-setup.yml | 2 +- .github/workflows/docker-unified.yml | 16 ++++++++-------- .github/workflows/publish-datahub-jars.yml | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index d266e1a7fd31f..f6320e1bd5c9f 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -43,7 +43,7 @@ jobs: timezoneLinux: ${{ matrix.timezone }} - uses: actions/checkout@v3 with: - fetch-depth: 0 + fetch-depth: 800 - name: Set up JDK 11 uses: actions/setup-java@v3 with: diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml index a8e6c611bc8f9..735ea1e56b169 100644 --- a/.github/workflows/check-datahub-jars.yml +++ b/.github/workflows/check-datahub-jars.yml @@ -38,7 +38,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - fetch-depth: 0 + fetch-depth: 800 - name: Set up JDK 11 uses: actions/setup-java@v3 with: diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index 43a8835a37a8c..6ce19a5b4616e 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -38,7 +38,7 @@ jobs: - name: Check out the repo uses: actions/checkout@v3 with: - fetch-depth: 0 + fetch-depth: 800 - uses: actions/setup-python@v4 with: python-version: "3.10" diff --git a/.github/workflows/docker-ingestion-base.yml b/.github/workflows/docker-ingestion-base.yml index 64f94ef3225ce..0d29f79aa5f6c 100644 --- a/.github/workflows/docker-ingestion-base.yml +++ b/.github/workflows/docker-ingestion-base.yml @@ -30,7 +30,7 @@ jobs: - name: Check out the repo uses: actions/checkout@v3 with: - fetch-depth: 0 + fetch-depth: 800 - name: Build and Push image uses: ./.github/actions/docker-custom-build-and-push with: diff --git a/.github/workflows/docker-ingestion-smoke.yml b/.github/workflows/docker-ingestion-smoke.yml index 70fcaacf8d12f..9e74f3a459378 100644 --- a/.github/workflows/docker-ingestion-smoke.yml +++ b/.github/workflows/docker-ingestion-smoke.yml @@ -53,7 +53,7 @@ jobs: - name: Check out the repo uses: actions/checkout@v3 with: - fetch-depth: 0 + fetch-depth: 800 - name: Build and push uses: ./.github/actions/docker-custom-build-and-push with: diff --git a/.github/workflows/docker-ingestion.yml b/.github/workflows/docker-ingestion.yml index 6716026851739..f3768cfde5002 100644 --- a/.github/workflows/docker-ingestion.yml +++ b/.github/workflows/docker-ingestion.yml @@ -57,7 +57,7 @@ jobs: - name: Check out the repo uses: actions/checkout@v3 with: - fetch-depth: 0 + fetch-depth: 800 - name: Build and push uses: ./.github/actions/docker-custom-build-and-push with: diff --git a/.github/workflows/docker-postgres-setup.yml b/.github/workflows/docker-postgres-setup.yml index ed2dc198aa6a8..a5d421d4b7ff5 100644 --- a/.github/workflows/docker-postgres-setup.yml +++ b/.github/workflows/docker-postgres-setup.yml @@ -48,7 +48,7 @@ jobs: - name: Check out the repo uses: actions/checkout@v3 with: - fetch-depth: 0 + fetch-depth: 800 - name: Build and push uses: ./.github/actions/docker-custom-build-and-push with: diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index 65028fd260807..1eb2a393600d2 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -64,7 +64,7 @@ jobs: - name: Check out the repo uses: actions/checkout@v3 with: - fetch-depth: 0 + fetch-depth: 800 - name: Pre-build artifacts for docker image run: | ./gradlew :metadata-service:war:build -x test --parallel @@ -122,7 +122,7 @@ jobs: - name: Check out the repo uses: actions/checkout@v3 with: - fetch-depth: 0 + fetch-depth: 800 - name: Pre-build artifacts for docker image run: | ./gradlew :metadata-jobs:mae-consumer-job:build -x test --parallel @@ -180,7 +180,7 @@ jobs: - name: Check out the repo uses: actions/checkout@v3 with: - fetch-depth: 0 + fetch-depth: 800 - name: Pre-build artifacts for docker image run: | ./gradlew :metadata-jobs:mce-consumer-job:build -x test --parallel @@ -238,7 +238,7 @@ jobs: - name: Check out the repo uses: actions/checkout@v3 with: - fetch-depth: 0 + fetch-depth: 800 - name: Pre-build artifacts for docker image run: | ./gradlew :datahub-upgrade:build -x test --parallel @@ -296,7 +296,7 @@ jobs: - name: Check out the repo uses: actions/checkout@v3 with: - fetch-depth: 0 + fetch-depth: 800 - name: Pre-build artifacts for docker image run: | ./gradlew :datahub-frontend:dist -x test -x yarnTest -x yarnLint --parallel @@ -356,7 +356,7 @@ jobs: - name: Check out the repo uses: actions/checkout@v3 with: - fetch-depth: 0 + fetch-depth: 800 - name: Build and push uses: ./.github/actions/docker-custom-build-and-push with: @@ -378,7 +378,7 @@ jobs: - name: Check out the repo uses: actions/checkout@v3 with: - fetch-depth: 0 + fetch-depth: 800 - name: Build and push uses: ./.github/actions/docker-custom-build-and-push with: @@ -400,7 +400,7 @@ jobs: - name: Check out the repo uses: actions/checkout@v3 with: - fetch-depth: 0 + fetch-depth: 800 - name: Build and push uses: ./.github/actions/docker-custom-build-and-push with: diff --git a/.github/workflows/publish-datahub-jars.yml b/.github/workflows/publish-datahub-jars.yml index 45fa677aede05..1a1523e8db1a1 100644 --- a/.github/workflows/publish-datahub-jars.yml +++ b/.github/workflows/publish-datahub-jars.yml @@ -50,7 +50,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - fetch-depth: 0 + fetch-depth: 800 - name: Set up JDK 11 uses: actions/setup-java@v3 with: From ef3b9489aa24d09ad4546c0645cdb2880158148a Mon Sep 17 00:00:00 2001 From: VISHAL KUMAR <110387730+vishalkSimplify@users.noreply.github.com> Date: Tue, 1 Aug 2023 19:34:35 +0530 Subject: [PATCH 213/222] feat(ingest/vertica): performance improvement and bug fixes (#8328) Co-authored-by: Harshal Sheth --- .../source/builder/RecipeForm/constants.ts | 27 +- .../source/builder/RecipeForm/vertica.ts | 119 + .../app/ingest/source/builder/sources.json | 2 +- .../docs/sources/vertica/README.md | 1 - .../docs/sources/vertica/vertica_pre.md | 2 +- .../docs/sources/vertica/vertica_recipe.yml | 1 - metadata-ingestion/setup.py | 9 +- .../datahub/ingestion/source/sql/vertica.py | 682 +- .../tests/integration/vertica/ddl.sql | 46 + .../integration/vertica/docker-compose.yml | 3 +- .../tests/integration/vertica/test_vertica.py | 49 +- .../vertica/vertica_mces_with_db_golden.json | 10232 +++++++++++++++- .../integration/vertica/vertica_to_file.yml | 2 +- 13 files changed, 10631 insertions(+), 544 deletions(-) create mode 100644 datahub-web-react/src/app/ingest/source/builder/RecipeForm/vertica.ts create mode 100644 metadata-ingestion/tests/integration/vertica/ddl.sql diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts index 44f48fa072cd8..351876fe6b16a 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts @@ -83,7 +83,7 @@ import { PROJECT_NAME, } from './lookml'; import { PRESTO, PRESTO_HOST_PORT, PRESTO_DATABASE, PRESTO_USERNAME, PRESTO_PASSWORD } from './presto'; -import { BIGQUERY_BETA, DBT_CLOUD, MYSQL, POWER_BI, UNITY_CATALOG } from '../constants'; +import { BIGQUERY_BETA, DBT_CLOUD, MYSQL, POWER_BI, UNITY_CATALOG, VERTICA } from '../constants'; import { BIGQUERY_BETA_PROJECT_ID, DATASET_ALLOW, DATASET_DENY, PROJECT_ALLOW, PROJECT_DENY } from './bigqueryBeta'; import { MYSQL_HOST_PORT, MYSQL_PASSWORD, MYSQL_USERNAME } from './mysql'; import { MSSQL, MSSQL_DATABASE, MSSQL_HOST_PORT, MSSQL_PASSWORD, MSSQL_USERNAME } from './mssql'; @@ -130,6 +130,17 @@ import { WORKSPACE_ID_DENY, } from './powerbi'; +import { + VERTICA_HOST_PORT, + VERTICA_DATABASE, + VERTICA_USERNAME, + VERTICA_PASSWORD, + INCLUDE_PROJECTIONS, + INCLUDE_MLMODELS, + INCLUDE_VIEW_LINEAGE, + INCLUDE_PROJECTIONS_LINEAGE, +} from './vertica'; + export enum RecipeSections { Connection = 0, Filter = 1, @@ -428,6 +439,20 @@ export const RECIPE_FIELDS: RecipeFields = { ], filterSectionTooltip: 'Include or exclude specific PowerBI Workspaces from ingestion.', }, + [VERTICA]: { + fields: [VERTICA_HOST_PORT, VERTICA_DATABASE, VERTICA_USERNAME, VERTICA_PASSWORD], + filterFields: [SCHEMA_ALLOW, SCHEMA_DENY, TABLE_ALLOW, TABLE_DENY, VIEW_ALLOW, VIEW_DENY], + advancedFields: [ + INCLUDE_TABLES, + INCLUDE_VIEWS, + INCLUDE_PROJECTIONS, + INCLUDE_MLMODELS, + INCLUDE_VIEW_LINEAGE, + INCLUDE_PROJECTIONS_LINEAGE, + TABLE_PROFILING_ENABLED, + ], + filterSectionTooltip: 'Include or exclude specific Schemas, Tables, Views and Projections from ingestion.', + }, }; export const CONNECTORS_WITH_FORM = new Set(Object.keys(RECIPE_FIELDS)); diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/vertica.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/vertica.ts new file mode 100644 index 0000000000000..ae475d152d750 --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/vertica.ts @@ -0,0 +1,119 @@ +import { get } from 'lodash'; +import { RecipeField, FieldType } from './common'; + +export const VERTICA_HOST_PORT: RecipeField = { + name: 'host_port', + label: 'Host and Port', + tooltip: + "The host and port where Vertica is running. For example, 'localhost:5433'. Note: this host must be accessible on the network where DataHub is running (or allowed via an IP Allow List, AWS PrivateLink, etc).", + type: FieldType.TEXT, + fieldPath: 'source.config.host_port', + placeholder: 'localhost:5433', + required: true, + rules: null, +}; + +export const VERTICA_DATABASE: RecipeField = { + name: 'database', + label: 'Database', + tooltip: 'Ingest metadata for a specific Database.', + type: FieldType.TEXT, + fieldPath: 'source.config.database', + placeholder: 'Vertica_Database', + required: true, + rules: null, +}; + +export const VERTICA_USERNAME: RecipeField = { + name: 'username', + label: 'Username', + tooltip: 'The Vertica username used to extract metadata.', + type: FieldType.TEXT, + fieldPath: 'source.config.username', + placeholder: 'Vertica_Username', + required: true, + rules: null, +}; + +export const VERTICA_PASSWORD: RecipeField = { + name: 'password', + label: 'Password', + tooltip: 'The Vertica password for the user.', + type: FieldType.SECRET, + fieldPath: 'source.config.password', + placeholder: 'Vertica_Password', + required: true, + rules: null, +}; + +const includeProjectionPath = 'source.config.include_projections'; +export const INCLUDE_PROJECTIONS: RecipeField = { + name: 'include_projections', + label: 'Include Projections', + tooltip: 'Extract Projections from source.', + type: FieldType.BOOLEAN, + fieldPath: includeProjectionPath, + // This is in accordance with what the ingestion sources do. + getValueFromRecipeOverride: (recipe: any) => { + const includeProjection = get(recipe, includeProjectionPath); + if (includeProjection !== undefined && includeProjection !== null) { + return includeProjection; + } + return true; + }, + rules: null, +}; + +const includemodelsPath = 'source.config.include_models'; +export const INCLUDE_MLMODELS: RecipeField = { + name: 'include_models', + label: 'Include ML Models', + tooltip: 'Extract ML models from source.', + type: FieldType.BOOLEAN, + fieldPath: includemodelsPath, + // This is in accordance with what the ingestion sources do. + getValueFromRecipeOverride: (recipe: any) => { + const includeModel = get(recipe, includemodelsPath); + if (includeModel !== undefined && includeModel !== null) { + return includeModel; + } + return true; + }, + rules: null, +}; + +const includeviewlineagePath = 'source.config.include_view_lineage'; +export const INCLUDE_VIEW_LINEAGE: RecipeField = { + name: 'include_view_lineage', + label: 'Include View Lineage', + tooltip: 'Extract View Lineage from source.', + type: FieldType.BOOLEAN, + fieldPath: includeviewlineagePath, + // This is in accordance with what the ingestion sources do. + getValueFromRecipeOverride: (recipe: any) => { + const includeviewlineage = get(recipe, includeviewlineagePath); + if (includeviewlineage !== undefined && includeviewlineage !== null) { + return includeviewlineage; + } + return true; + }, + rules: null, +}; + +const includeprojectionlineagePath = 'source.config.include_projection_lineage'; +export const INCLUDE_PROJECTIONS_LINEAGE: RecipeField = { + name: 'include_projection_lineage', + label: 'Include Projection Lineage', + tooltip: 'Extract Projection Lineage from source.', + type: FieldType.BOOLEAN, + fieldPath: includeprojectionlineagePath, + // This is in accordance with what the ingestion sources do. + getValueFromRecipeOverride: (recipe: any) => { + const includeprojectionlineage = get(recipe, includeprojectionlineagePath); + if (includeprojectionlineage !== undefined && includeprojectionlineage !== null) { + return includeprojectionlineage; + } + return true; + }, + rules: null, +}; diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json index c9db0433b3aae..13643c58f72e1 100644 --- a/datahub-web-react/src/app/ingest/source/builder/sources.json +++ b/datahub-web-react/src/app/ingest/source/builder/sources.json @@ -200,7 +200,7 @@ "name": "vertica", "displayName": "Vertica", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/vertica/", - "recipe": "source:\n type: vertica\n config:\n # Coordinates\n host_port: localhost:5433\n # The name of the vertica database\n database: Vmart\n # Credentials\n username: dbadmin\n password:null\n include_tables: true\n include_views: true\n include_projections: true\n include_oauth: true\n include_models: true\n include_view_lineage: true\n include_projection_lineage: true\n profiling:\n enabled: true\n stateful_ingestion:\n enabled: true " + "recipe": "source:\n type: vertica\n config:\n # Coordinates\n host_port: localhost:5433\n # The name of the vertica database\n database: Database_Name\n # Credentials\n username: Vertica_User\n password: Vertica_Password\n\n include_tables: true\n include_views: true\n include_projections: true\n include_models: true\n include_view_lineage: true\n include_projection_lineage: true\n profiling:\n enabled: false\n stateful_ingestion:\n enabled: true " }, { "urn": "urn:li:dataPlatform:custom", diff --git a/metadata-ingestion/docs/sources/vertica/README.md b/metadata-ingestion/docs/sources/vertica/README.md index 3c41b744b321f..fb016fbdb2d28 100644 --- a/metadata-ingestion/docs/sources/vertica/README.md +++ b/metadata-ingestion/docs/sources/vertica/README.md @@ -8,7 +8,6 @@ The DataHub Vertica Plugin extracts the following: * Metadata for databases, schemas, views, tables, and projections * Table level lineage * Metadata for ML Models -* Metadata for Vertica OAuth ### Concept Mapping diff --git a/metadata-ingestion/docs/sources/vertica/vertica_pre.md b/metadata-ingestion/docs/sources/vertica/vertica_pre.md index dbcda212cf02f..a4e71de3c908c 100644 --- a/metadata-ingestion/docs/sources/vertica/vertica_pre.md +++ b/metadata-ingestion/docs/sources/vertica/vertica_pre.md @@ -2,5 +2,5 @@ In order to ingest metadata from Vertica, you will need: -- Vertica Server Version 10.1.1-0 and avobe. It may also work for older versions. +- Vertica Server Version 10.1.1-0 and above. It may also work with, but is not been tested with, older versions . - Vertica Credentials (Username/Password) diff --git a/metadata-ingestion/docs/sources/vertica/vertica_recipe.yml b/metadata-ingestion/docs/sources/vertica/vertica_recipe.yml index 88ac74cc22c5f..ad4f0173b0111 100644 --- a/metadata-ingestion/docs/sources/vertica/vertica_recipe.yml +++ b/metadata-ingestion/docs/sources/vertica/vertica_recipe.yml @@ -12,7 +12,6 @@ source: include_tables: true include_views: true include_projections: true - include_oauth: true include_models: true include_view_lineage: true include_projection_lineage: true diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index d6738f4b9782c..9940e8cdd0a97 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -374,7 +374,9 @@ def get_long_description(): "nifi": {"requests", "packaging", "requests-gssapi"}, "powerbi": microsoft_common | {"lark[regex]==1.1.4", "sqlparse"}, "powerbi-report-server": powerbi_report_server, - "vertica": sql_common | {"vertica-sqlalchemy-dialect[vertica-python]==0.0.1"}, + + "vertica": sql_common | {"vertica-sqlalchemy-dialect[vertica-python]==0.0.8"}, + "unity-catalog": databricks | sqllineage_lib, } @@ -488,7 +490,8 @@ def get_long_description(): "powerbi-report-server", "salesforce", "unity-catalog", - "nifi" + "nifi", + "vertica" # airflow is added below ] if plugin @@ -522,7 +525,7 @@ def get_long_description(): "mysql", "mariadb", "redash", - # "vertica", + "vertica", ] for dependency in plugins[plugin] ), diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py index 37ccc1d1fedb5..3b70c476639a9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py @@ -1,15 +1,11 @@ import logging import traceback -from collections import defaultdict from dataclasses import dataclass -from textwrap import dedent from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Set, Tuple, Union import pydantic from pydantic.class_validators import validator -from sqlalchemy import create_engine, sql -from sqlalchemy.engine.reflection import Inspector -from sqlalchemy.exc import ProgrammingError +from vertica_sqlalchemy_dialect.base import VerticaInspector from datahub.configuration.common import AllowDenyPattern from datahub.emitter.mce_builder import ( @@ -63,24 +59,21 @@ class VerticaSourceReport(SQLSourceReport): projection_scanned: int = 0 models_scanned: int = 0 - oauth_scanned: int = 0 def report_entity_scanned(self, name: str, ent_type: str = "table") -> None: """ - Entity could be a projection or a models or Oauth . + Entity could be a projection or a model. """ if ent_type == "projection": self.projection_scanned += 1 elif ent_type == "models": self.models_scanned += 1 - elif ent_type == "oauth": - self.oauth_scanned += 1 else: super().report_entity_scanned(name, ent_type) -# Extended BasicSQLAlchemyConfig to config for projections,models and oauth metadata. +# Extended BasicSQLAlchemyConfig to config for projections,models metadata. class VerticaConfig(BasicSQLAlchemyConfig): models_pattern: AllowDenyPattern = pydantic.Field( default=AllowDenyPattern.allow_all(), @@ -92,13 +85,12 @@ class VerticaConfig(BasicSQLAlchemyConfig): include_models: Optional[bool] = pydantic.Field( default=True, description="Whether Models should be ingested." ) - include_oauth: Optional[bool] = pydantic.Field( - default=True, description="Whether Oauth should be ingested." - ) + include_view_lineage: Optional[bool] = pydantic.Field( default=True, description="If the source supports it, include view lineage to the underlying storage location.", ) + include_projection_lineage: Optional[bool] = pydantic.Field( default=True, description="If the source supports it, include view lineage to the underlying storage location.", @@ -132,10 +124,6 @@ def __init__(self, config: VerticaConfig, ctx: PipelineContext): # self.platform = platform super(VerticaSource, self).__init__(config, ctx, "vertica") self.report: SQLSourceReport = VerticaSourceReport() - self.view_lineage_map: Optional[Dict[str, List[Tuple[str, str, str]]]] = None - self.projection_lineage_map: Optional[ - Dict[str, List[Tuple[str, str, str]]] - ] = None self.config: VerticaConfig = config @classmethod @@ -201,16 +189,13 @@ def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit profile_requests, profiler, platform=self.platform ) - oauth_schema = "Entities" - if sql_config.include_oauth: - yield from self.loop_oauth(inspector, oauth_schema, sql_config) - def get_database_properties( - self, inspector: Inspector, database: str + self, inspector: VerticaInspector, database: str ) -> Optional[Dict[str, str]]: try: - custom_properties = inspector._get_database_properties(database) # type: ignore + custom_properties = inspector._get_database_properties(database) return custom_properties + except Exception as ex: self.report.report_failure( f"{database}", f"unable to get extra_properties : {ex}" @@ -218,10 +203,10 @@ def get_database_properties( return None def get_schema_properties( - self, inspector: Inspector, database: str, schema: str + self, inspector: VerticaInspector, database: str, schema: str ) -> Optional[Dict[str, str]]: try: - custom_properties = inspector._get_schema_properties(schema) # type: ignore + custom_properties = inspector._get_schema_properties(schema) return custom_properties except Exception as ex: self.report.report_failure( @@ -232,7 +217,7 @@ def get_schema_properties( def _process_table( self, dataset_name: str, - inspector: Inspector, + inspector: VerticaInspector, schema: str, table: str, sql_config: SQLAlchemyConfig, @@ -243,7 +228,7 @@ def _process_table( self.config.platform_instance, self.config.env, ) - table_owner = self._get_owner_information(table, "table") + table_owner = inspector.get_table_owner(table, schema) yield from add_owner_to_entity_wu( entity_type="dataset", entity_urn=dataset_urn, @@ -255,7 +240,7 @@ def _process_table( def loop_views( self, - inspector: Inspector, + inspector: VerticaInspector, schema: str, sql_config: SQLAlchemyConfig, ) -> Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]: @@ -286,7 +271,7 @@ def loop_views( self.report.report_warning( f"{schema}.{view}", f"Ingestion error: {e}" ) - if sql_config.include_view_lineage: # type: ignore + if self.config.include_view_lineage: try: dataset_urn = make_dataset_urn_with_platform_instance( self.platform, @@ -294,19 +279,22 @@ def loop_views( self.config.platform_instance, self.config.env, ) + dataset_snapshot = DatasetSnapshot( urn=dataset_urn, aspects=[StatusClass(removed=False)] ) + lineage_info = self._get_upstream_lineage_info( - dataset_urn, view + dataset_urn, + inspector, + view, + schema, ) + if lineage_info is not None: # upstream_column_props = [] yield MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, entityUrn=dataset_snapshot.urn, - aspectName="upstreamLineage", aspect=lineage_info, ).as_workunit() @@ -323,7 +311,7 @@ def loop_views( def _process_view( self, dataset_name: str, - inspector: Inspector, + inspector: VerticaInspector, schema: str, view: str, sql_config: SQLAlchemyConfig, @@ -334,7 +322,7 @@ def _process_view( Args: dataset_name (str) inspector (Inspector) - schema (str): schame name + schema (str): schema name view (str): name of the view to inspect sql_config (SQLAlchemyConfig) table_tags (Dict[str, str], optional) Defaults to dict(). @@ -345,25 +333,28 @@ def _process_view( Yields: Iterator[Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]] """ + dataset_urn = make_dataset_urn_with_platform_instance( self.platform, dataset_name, self.config.platform_instance, self.config.env, ) - view_owner = self._get_owner_information(view, "view") + + view_owner = inspector.get_view_owner(view, schema) yield from add_owner_to_entity_wu( entity_type="dataset", entity_urn=dataset_urn, owner_urn=f"urn:li:corpuser:{view_owner}", ) + yield from super()._process_view( dataset_name, inspector, schema, view, sql_config ) def loop_projections( self, - inspector: Inspector, + inspector: VerticaInspector, schema: str, sql_config: SQLAlchemyConfig, ) -> Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]: @@ -386,8 +377,7 @@ def loop_projections( projections_seen: Set[str] = set() try: - # table_tags = self.get_extra_tags(inspector, schema, "projection") - for projection in inspector.get_projection_names(schema): # type: ignore + for projection in inspector.get_projection_names(schema): dataset_name = self.get_identifier( schema=schema, entity=projection, inspector=inspector ) @@ -412,7 +402,7 @@ def loop_projections( self.report.report_warning( f"{schema}.{projection}", f"Ingestion error: {ex}" ) - if sql_config.include_projection_lineage: # type: ignore + if self.config.include_projection_lineage: try: dataset_urn = make_dataset_urn_with_platform_instance( self.platform, @@ -422,37 +412,35 @@ def loop_projections( ) dataset_snapshot = DatasetSnapshot( - urn=dataset_urn, - aspects=[StatusClass(removed=False)], + urn=dataset_urn, aspects=[StatusClass(removed=False)] ) + lineage_info = self._get_upstream_lineage_info_projection( - dataset_urn, projection + dataset_urn, inspector, projection, schema ) if lineage_info is not None: yield MetadataChangeProposalWrapper( entityUrn=dataset_snapshot.urn, aspect=lineage_info ).as_workunit() - except Exception as ex: + + except Exception as e: logger.warning( - f"Unable to get lineage of Projection {schema}.{projection} due to an exception %s", - ex, - ) - self.report.report_warning( - f"{schema}.{projection}", f"Ingestion error: {ex}" + f"Unable to get lineage of projection {projection} due to an exception.\n {traceback.format_exc()}" ) + self.report.report_warning(f"{schema}", f"Ingestion error: {e}") except Exception as ex: self.report.report_failure(f"{schema}", f"Projection error: {ex}") def _process_projections( self, dataset_name: str, - inspector: Inspector, + inspector: VerticaInspector, schema: str, projection: str, sql_config: SQLAlchemyConfig, ) -> Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]: - columns = self._get_columns(dataset_name, inspector, schema, projection) + columns = inspector.get_projection_columns(projection, schema) dataset_urn = make_dataset_urn_with_platform_instance( self.platform, dataset_name, @@ -474,20 +462,7 @@ def _process_projections( ) dataset_snapshot.aspects.append(dataset_properties) - if location_urn: - external_upstream_table = UpstreamClass( - dataset=location_urn, - type=DatasetLineageTypeClass.COPY, - ) - yield MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, - entityUrn=dataset_snapshot.urn, - aspectName="upstreamLineage", - aspect=UpstreamLineage(upstreams=[external_upstream_table]), - ).as_workunit() - - projection_owner = self._get_owner_information(projection, "projection") + projection_owner = inspector.get_projection_owner(projection, schema) yield from add_owner_to_entity_wu( entity_type="dataset", entity_urn=dataset_urn, @@ -533,49 +508,87 @@ def _process_projections( domain_registry=self.domain_registry, ) - def get_projection_properties( - self, inspector: Inspector, schema: str, projection: str - ) -> Tuple[Optional[str], Dict[str, str], Optional[str]]: - """ - Returns projection related metadata information to show in properties tab - eg. projection type like super, segmented etc - partition key, segmentation and so on + def loop_profiler_requests( + self, + inspector: VerticaInspector, + schema: str, + sql_config: SQLAlchemyConfig, + ) -> Iterable["GEProfilerRequest"]: + """Function is used for collecting profiling related information for every projections + inside an schema. + + We have extended original loop_profiler_requests and added functionality for projection + profiling . All the logics are same only change is we have ran the loop to get projection name from inspector.get_projection_name . + + Args: schema: schema name - Args: - inspector (Inspector): inspector obj from reflection engine - schema (str): schema name - projection (str): projection name - Returns: - Tuple[Optional[str], Dict[str, str], Optional[str]]: [description] """ - description: Optional[str] = None - properties: Dict[str, str] = {} - # The location cannot be fetched generically, but subclasses may override - # this method and provide a location. - location: Optional[str] = None - try: - # SQLAlchemy stubs are incomplete and missing this method. - # PR: https://github.com/dropbox/sqlalchemy-stubs/pull/223. - projection_info: dict = inspector.get_projection_comment(projection, schema) # type: ignore - except NotImplementedError: - return description, properties, location - except ProgrammingError as error: + from datahub.ingestion.source.ge_data_profiler import GEProfilerRequest + + tables_seen: Set[str] = set() + profile_candidates = None # Default value if profile candidates not available. + yield from super().loop_profiler_requests(inspector, schema, sql_config) + + for projection in inspector.get_projection_names(schema): + dataset_name = self.get_identifier( + schema=schema, entity=projection, inspector=inspector + ) + + if not self.is_dataset_eligible_for_profiling( + dataset_name, sql_config, inspector, profile_candidates + ): + if self.config.profiling.report_dropped_profiles: + self.report.report_dropped(f"profile of {dataset_name}") + continue + if dataset_name not in tables_seen: + tables_seen.add(dataset_name) + else: + logger.debug(f"{dataset_name} has already been seen, skipping...") + continue + missing_column_info_warn = self.report.warnings.get(MISSING_COLUMN_INFO) + if ( + missing_column_info_warn is not None + and dataset_name in missing_column_info_warn + ): + continue + (partition, custom_sql) = self.generate_partition_profiler_query( + schema, projection, self.config.profiling.partition_datetime + ) + if partition is None and self.is_table_partitioned( + database=None, schema=schema, table=projection + ): + self.report.report_warning( + "profile skipped as partitioned table is empty or partition id was invalid", + dataset_name, + ) + continue + if ( + partition is not None + and not self.config.profiling.partition_profiling_enabled + ): + logger.debug( + f"{dataset_name} and partition {partition} is skipped because profiling.partition_profiling_enabled property is disabled" + ) + continue + self.report.report_entity_profiled(dataset_name) logger.debug( - f"Encountered ProgrammingError. Retrying with quoted schema name for schema {schema} and table {properties} %s", - error, + f"Preparing profiling request for {schema}, {projection}, {partition}" + ) + + yield GEProfilerRequest( + pretty_name=dataset_name, + batch_kwargs=self.prepare_profiler_args( + inspector=inspector, + schema=schema, + table=projection, + partition=partition, + custom_sql=custom_sql, + ), ) - projection_info: dict = inspector.get_projection_comment(properties, f'"{schema}"') # type: ignore - description = projection_info.get("text") - if isinstance(description, tuple): - # Handling for value type tuple which is coming for dialect 'db2+ibm_db' - description = projection_info["text"][0] - # The "properties" field is a non-standard addition to SQLAlchemy's interface. - properties = projection_info.get("properties", {}) - return description, properties, location def loop_models( self, - inspector: Inspector, + inspector: VerticaInspector, schema: str, sql_config: SQLAlchemyConfig, ) -> Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]: @@ -595,7 +608,7 @@ def loop_models( """ models_seen: Set[str] = set() try: - for models in inspector.get_models_names(schema): # type: ignore + for models in inspector.get_models_names(schema): dataset_name = self.get_identifier( schema="Entities", entity=models, inspector=inspector ) @@ -630,7 +643,7 @@ def loop_models( def _process_models( self, dataset_name: str, - inspector: Inspector, + inspector: VerticaInspector, schema: str, table: str, sql_config: SQLAlchemyConfig, @@ -689,10 +702,7 @@ def _process_models( if dpi_aspect: yield dpi_aspect yield MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, entityUrn=dataset_urn, - aspectName="subTypes", aspect=SubTypesClass(typeNames=["ML Models"]), ).as_workunit() if self.config.domain: @@ -704,19 +714,20 @@ def _process_models( domain_registry=self.domain_registry, ) - def get_model_properties( - self, inspector: Inspector, schema: str, model: str + def get_projection_properties( + self, inspector: VerticaInspector, schema: str, projection: str ) -> Tuple[Optional[str], Dict[str, str], Optional[str]]: """ - Returns ml models related metadata information to show in properties tab - eg. ml model attribute and ml model specification information. + Returns projection related metadata information to show in properties tab + eg. projection type like super, segmented etc + partition key, segmentation and so on Args: inspector (Inspector): inspector obj from reflection engine schema (str): schema name - model (str): ml model name + projection (str): projection name Returns: - Tuple[Optional[str], Dict[str, str], Optional[str]] + Tuple[Optional[str], Dict[str, str], Optional[str]]: [description] """ description: Optional[str] = None properties: Dict[str, str] = {} @@ -724,154 +735,28 @@ def get_model_properties( # this method and provide a location. location: Optional[str] = None try: - table_info: dict = inspector.get_model_comment(model, schema) # type: ignore + # SQLAlchemy stubs are incomplete and missing this method. + # PR: https://github.com/dropbox/sqlalchemy-stubs/pull/223. + projection_info: dict = inspector.get_projection_comment(projection, schema) except NotImplementedError: return description, properties, location - except ProgrammingError as error: - logger.debug( - f"Encountered ProgrammingError. Retrying with quoted schema name for schema {schema} and {model} %s", - error, - ) - table_info: dict = inspector.get_model_comment(model, f'"{schema}"') # type: ignore - description = table_info.get("text") - if type(description) is tuple: + description = projection_info.get("text") + if isinstance(description, tuple): # Handling for value type tuple which is coming for dialect 'db2+ibm_db' - description = table_info["text"][0] + description = projection_info["text"][0] # The "properties" field is a non-standard addition to SQLAlchemy's interface. - properties = table_info.get("properties", {}) + properties = projection_info.get("properties", {}) return description, properties, location - def loop_oauth( - self, - inspector: Inspector, - schema: str, - sql_config: SQLAlchemyConfig, - ) -> Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]: - """ - This function is for iterating over the oauth in vertica db - Args: - inspector (Inspector): - schema (str): schema name - sql_config (SQLAlchemyConfig): configuration - - Returns: - Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]: [description] - """ - oauth_seen: Set[str] = set() - try: - for oauth in inspector.get_Oauth_names(schema): # type: ignore - dataset_name = self.get_identifier( - schema=schema, entity=oauth, inspector=inspector - ) - - if dataset_name not in oauth_seen: - oauth_seen.add(dataset_name) - else: - logger.debug("has already been seen, skipping %s", dataset_name) - continue - self.report.report_entity_scanned(dataset_name, ent_type="oauth") - if not sql_config.table_pattern.allowed(dataset_name): - self.report.report_dropped(dataset_name) - continue - try: - yield from self._process_oauth( - dataset_name, - inspector, - schema, - oauth, - sql_config, - ) - except Exception as error: - self.report.report_warning( - f"{schema}.{oauth}", f"Ingestion error: {error}" - ) - except Exception as error: - self.report.report_failure(f"{schema}", f"oauth error: {error}") - - def _process_oauth( - self, - dataset_name: str, - inspector: Inspector, - schema: str, - oauth: str, - sql_config: SQLAlchemyConfig, - ) -> Iterable[Union[SqlWorkUnit, MetadataWorkUnit]]: - """ - To fetch oauth related metadata of oauth from vertica db - Args: - dataset_name (str): dataset name - inspector (Inspector): inspector object from reflection - schema (str): schema name - Oauth (str): oauth name - sql_config (SQLAlchemyConfig): configuration - Returns: - Iterable[Union[SqlWorkUnit, MetadataWorkUnit]] - """ - columns: List[Dict[Any, Any]] = [] - dataset_urn = make_dataset_urn_with_platform_instance( - self.platform, - dataset_name, - self.config.platform_instance, - self.config.env, - ) - dataset_snapshot = DatasetSnapshot( - urn=dataset_urn, - aspects=[StatusClass(removed=False)], - ) - description, properties, location_urn = self.get_oauth_properties( - inspector, schema, oauth - ) - - dataset_properties = DatasetPropertiesClass( - name=oauth, - description=description, - customProperties=properties, - ) - dataset_snapshot.aspects.append(dataset_properties) - - schema_fields = self.get_schema_fields(dataset_name, columns) - schema_metadata = get_schema_metadata( - self.report, - dataset_name, - self.platform, - columns, - schema_fields, # type: ignore - ) - dataset_snapshot.aspects.append(schema_metadata) - db_name = self.get_db_name(inspector) - - yield from self.add_table_to_schema_container(dataset_urn, db_name, schema) - mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot) - yield SqlWorkUnit(id=dataset_name, mce=mce) - dpi_aspect = self.get_dataplatform_instance_aspect(dataset_urn=dataset_urn) - if dpi_aspect: - yield dpi_aspect - yield MetadataChangeProposalWrapper( - entityType="dataset", - changeType=ChangeTypeClass.UPSERT, - entityUrn=dataset_urn, - aspectName="subTypes", - aspect=SubTypesClass(typeNames=["oauth"]), - ).as_workunit() - - if self.config.domain: - assert self.domain_registry - yield from get_domain_wu( - dataset_name=dataset_name, - entity_urn=dataset_urn, - domain_config=self.config.domain, - domain_registry=self.domain_registry, - ) - - def get_oauth_properties( - self, inspector: Inspector, schema: str, model: str + def get_model_properties( + self, inspector: VerticaInspector, schema: str, model: str ) -> Tuple[Optional[str], Dict[str, str], Optional[str]]: """ - Returns oauth related metadata information to show in properties tab - eg. is_auth_enabled , auth_priority and all auth related info. + Returns ml models related metadata information to show in properties tab + eg. ml model attribute and ml model specification information. Args: - inspector (Inspector): inspector obj from reflection engine + inspector (VerticaInspector): inspector obj from reflection engine schema (str): schema name model (str): ml model name Returns: @@ -883,110 +768,39 @@ def get_oauth_properties( # this method and provide a location. location: Optional[str] = None try: - table_info: dict = inspector.get_oauth_comment(model, schema) # type: ignore + table_info: dict = inspector.get_model_comment(model, schema) except NotImplementedError: return description, properties, location - except ProgrammingError as error: - logger.debug( - f"Encountered ProgrammingError. Retrying with quoted schema name for schema {schema} and oauth {model} %s", - error, - ) - table_info: dict = inspector.get_oauth_comment(model, f'"{schema}"') # type: ignore description = table_info.get("text") - if isinstance(description, tuple): - # Handling for value type tuple which is coming for dialect 'db2+ibm_db' - description = table_info["text"][0] + # The "properties" field is a non-standard addition to SQLAlchemy's interface. properties = table_info.get("properties", {}) return description, properties, location - def loop_profiler_requests( + def _get_upstream_lineage_info( self, - inspector: Inspector, + dataset_urn: str, + inspector: VerticaInspector, + view: str, schema: str, - sql_config: SQLAlchemyConfig, - ) -> Iterable["GEProfilerRequest"]: - """Function is used for collecting profiling related information for every projections - inside an schema. - - Args: schema: schema name - - """ - from datahub.ingestion.source.ge_data_profiler import GEProfilerRequest - - tables_seen: Set[str] = set() - profile_candidates = None # Default value if profile candidates not available. - yield from super().loop_profiler_requests(inspector, schema, sql_config) - for projection in inspector.get_projection_names(schema): # type: ignore - dataset_name = self.get_identifier( - schema=schema, entity=projection, inspector=inspector - ) - - if not self.is_dataset_eligible_for_profiling( - dataset_name, sql_config, inspector, profile_candidates - ): - if self.config.profiling.report_dropped_profiles: - self.report.report_dropped(f"profile of {dataset_name}") - continue - if dataset_name not in tables_seen: - tables_seen.add(dataset_name) - else: - logger.debug(f"{dataset_name} has already been seen, skipping...") - continue - missing_column_info_warn = self.report.warnings.get(MISSING_COLUMN_INFO) - if ( - missing_column_info_warn is not None - and dataset_name in missing_column_info_warn - ): - continue - (partition, custom_sql) = self.generate_partition_profiler_query( - schema, projection, self.config.profiling.partition_datetime - ) - if partition is None and self.is_table_partitioned( - database=None, schema=schema, table=projection - ): - self.report.report_warning( - "profile skipped as partitioned table is empty or partition id was invalid", - dataset_name, - ) - continue - if ( - partition is not None - and not self.config.profiling.partition_profiling_enabled - ): - logger.debug( - f"{dataset_name} and partition {partition} is skipped because profiling.partition_profiling_enabled property is disabled" - ) - continue - self.report.report_entity_profiled(dataset_name) - logger.debug( - f"Preparing profiling request for {schema}, {projection}, {partition}" - ) - - yield GEProfilerRequest( - pretty_name=dataset_name, - batch_kwargs=self.prepare_profiler_args( - inspector=inspector, - schema=schema, - table=projection, - partition=partition, - custom_sql=custom_sql, - ), - ) - - def _get_upstream_lineage_info( - self, dataset_urn: str, view: str ) -> Optional[_Aspect]: dataset_key = dataset_urn_to_key(dataset_urn) if dataset_key is None: logger.warning(f"Invalid dataset urn {dataset_urn}. Could not get key!") return None - self._populate_view_lineage(view) - dataset_name = dataset_key.name - lineage = self.view_lineage_map[dataset_name] # type: ignore + view_lineage_map = inspector._populate_view_lineage(view, schema) + if dataset_key.name is not None: + dataset_name = dataset_key.name - if not (lineage): + else: + # Handle the case when dataset_key.name is None + # You can raise an exception, log a warning, or take any other appropriate action + logger.warning("Invalid dataset name") + + lineage = view_lineage_map[dataset_name] + + if lineage is None: logger.debug(f"No lineage found for {dataset_name}") return None upstream_tables: List[UpstreamClass] = [] @@ -1015,87 +829,21 @@ def _get_upstream_lineage_info( return None - def _populate_view_lineage(self, view: str) -> None: - """Collects upstream and downstream lineage information for views . - - Args: - view (str): name of the view - - """ - - url = self.config.get_sql_alchemy_url() - engine = create_engine(url, **self.config.options) - - get_refrence_table = sql.text( - dedent( - """ select reference_table_name from v_catalog.view_tables where table_name = '%(view)s' """ - % {"view": view} - ) - ) - - refrence_table = "" - for data in engine.execute(get_refrence_table): - # refrence_table.append(data) - refrence_table = data["reference_table_name"] - - view_upstream_lineage_query = sql.text( - dedent( - """ - select reference_table_name ,reference_table_schema from v_catalog.view_tables where table_name = '%(view)s' """ - % {"view": view} - ) - ) - - view_downstream_query = sql.text( - dedent( - """ - select table_name ,table_schema from v_catalog.view_tables where reference_table_name = '%(view)s' - """ - % {"view": refrence_table} - ) - ) - num_edges: int = 0 - - try: - self.view_lineage_map = defaultdict(list) - for db_row_key in engine.execute(view_downstream_query): - downstream = f"{db_row_key['table_schema']}.{db_row_key['table_name']}" - - for db_row_value in engine.execute(view_upstream_lineage_query): - upstream = f"{db_row_value['reference_table_schema']}.{db_row_value['reference_table_name']}" - - view_upstream: str = upstream - view_name: str = downstream - self.view_lineage_map[view_name].append( - # (, , ) - (view_upstream, "[]", "[]") - ) - - num_edges += 1 - - except Exception as e: - self.warn( - logger, - "view_upstream_lineage", - "Extracting the upstream & Downstream view lineage from vertica failed." - + f"Please check your permissions. Continuing...\nError was {e}.", - ) - - logger.info( - f"A total of {num_edges} View upstream edges found found for {view}" - ) - def _get_upstream_lineage_info_projection( - self, dataset_urn: str, projection: str + self, + dataset_urn: str, + inspector: VerticaInspector, + projection: str, + schema: str, ) -> Optional[_Aspect]: dataset_key = dataset_urn_to_key(dataset_urn) if dataset_key is None: logger.warning(f"Invalid dataset urn {dataset_urn}. Could not get key!") return None - self._populate_projection_lineage(projection) + projection_lineage = inspector._populate_projection_lineage(projection, schema) dataset_name = dataset_key.name - lineage = self.projection_lineage_map[dataset_name] # type: ignore + lineage = projection_lineage[dataset_name] if not (lineage): logger.debug(f"No lineage found for {dataset_name}") @@ -1103,7 +851,7 @@ def _get_upstream_lineage_info_projection( upstream_tables: List[UpstreamClass] = [] for lineage_entry in lineage: - # Update the projection-lineage + # Update the view-lineage upstream_table_name = lineage_entry[0] upstream_table = UpstreamClass( @@ -1119,115 +867,9 @@ def _get_upstream_lineage_info_projection( if upstream_tables: logger.debug( - f"lineage of Projection '{dataset_name}': {[u.dataset for u in upstream_tables]}" - ) - return UpstreamLineage(upstreams=upstream_tables) - return None - - def _populate_projection_lineage(self, projection: str) -> None: - """ - Collects upstream and downstream lineage information for views . - - Args: - projection (str): name of the projection - - - """ - url = self.config.get_sql_alchemy_url() - logger.debug(f"sql_alchemy_url={url}") - engine = create_engine(url, **self.config.options) - - view_upstream_lineage_query = sql.text( - dedent( - """ select basename , schemaname from vs_projections where name ='%(projection)s' - """ - % {"projection": projection} - ) - ) - - num_edges: int = 0 - - try: - self.projection_lineage_map = defaultdict(list) - for db_row_key in engine.execute(view_upstream_lineage_query): - basename = db_row_key["basename"] - upstream = f"{db_row_key['schemaname']}.{db_row_key['basename']}" - - view_downstream_query = sql.text( - dedent( - """ - select name,schemaname - from vs_projections - where basename='%(basename)s' - """ - % {"basename": basename} - ) - ) - for db_row_value in engine.execute(view_downstream_query): - downstream = f"{db_row_value['schemaname']}.{db_row_value['name']}" - projection_upstream: str = upstream - projection_name: str = downstream - self.projection_lineage_map[projection_name].append( - # (, , ) - (projection_upstream, "[]", "[]") - ) - num_edges += 1 - - except Exception as error: - logger.warning( - "Extracting the upstream & downstream Projection lineage from Vertica failed %s", - error, - ) - - logger.info( - f"A total of {num_edges} Projection lineage edges found for {projection}." - ) - - def _get_owner_information(self, table: str, label: str) -> Optional[str]: - url = self.config.get_sql_alchemy_url() - engine = create_engine(url, **self.config.options) - if label == "table": - get_owner_query = sql.text( - dedent( - """ - SELECT owner_name - FROM v_catalog.tables - WHERE table_name = '%(table)s' - """ - % {"table": table} - ) - ) - - for each in engine.execute(get_owner_query): - return each["owner_name"] - elif label == "view": - get_owner_query = sql.text( - dedent( - """ - SELECT owner_name - FROM v_catalog.views - WHERE table_name = '%(view)s' - """ - % {"view": table} - ) - ) - - for each in engine.execute(get_owner_query): - return each["owner_name"] - - elif label == "projection": - get_owner_query = sql.text( - dedent( - """ - SELECT owner_name - FROM v_catalog.projections - WHERE projection_name = '%(projection)s' - """ - % {"projection": table} - ) + f" lineage of '{dataset_name}': {[u.dataset for u in upstream_tables]}" ) - for each in engine.execute(get_owner_query): - return each["owner_name"] + return UpstreamLineage(upstreams=upstream_tables) return None diff --git a/metadata-ingestion/tests/integration/vertica/ddl.sql b/metadata-ingestion/tests/integration/vertica/ddl.sql new file mode 100644 index 0000000000000..59a71a1a1f7b5 --- /dev/null +++ b/metadata-ingestion/tests/integration/vertica/ddl.sql @@ -0,0 +1,46 @@ +\set AUTOCOMMIT on +ALTER USER dbadmin IDENTIFIED BY 'abc123'; + +-- Create a Top-k projection +CREATE TABLE readings (meter_id INT, reading_date TIMESTAMP, reading_value FLOAT); +CREATE PROJECTION readings_topk (meter_id, recent_date, recent_value) AS SELECT meter_id, reading_date, reading_value FROM readings LIMIT 5 OVER (PARTITION BY meter_id ORDER BY reading_date DESC); + +-- Create a live agg projs +CREATE TABLE clicks(user_id IDENTITY(1,1), page_id INTEGER, click_time TIMESTAMP NOT NULL); +CREATE PROJECTION clicks_agg AS SELECT page_id, click_time::DATE click_date, COUNT(*) num_clicks FROM clicks GROUP BY page_id, click_time::DATE; + + +-- Create a VIEW +CREATE VIEW sampleview AS SELECT SUM(annual_income), customer_state +FROM public.customer_dimension +WHERE customer_key IN (SELECT customer_key FROM store.store_sales_fact) +GROUP BY customer_state ORDER BY customer_state ASC; + +-- Step 1: Create library +\set libfile '\''/opt/sqlalchemy-vertica-dialect'/python/TransformFunctions.py\'' +CREATE LIBRARY TransformFunctions AS '/opt/vertica/sdk/examples/python/TransformFunctions.py' LANGUAGE 'Python'; + + +-- Step 2: Create functions +CREATE TRANSFORM FUNCTION tokenize AS NAME 'StringTokenizerFactory' LIBRARY TransformFunctions; +CREATE TRANSFORM FUNCTION topk AS NAME 'TopKPerPartitionFactory' LIBRARY TransformFunctions; + +CREATE TABLE phrases (phrase VARCHAR(128)); +COPY phrases FROM STDIN; +Word +The quick brown fox jumped over the lazy dog +\. + +SELECT tokenize(phrase) OVER () FROM phrases; + +-- Create a temp table + +CREATE TEMPORARY TABLE sampletemp (a int, b int) ON COMMIT PRESERVE ROWS; +INSERT INTO sampletemp VALUES(1,2); + +-- Create partition key +ALTER TABLE store.store_orders_fact PARTITION BY date_ordered::DATE GROUP BY DATE_TRUNC('month', (date_ordered)::DATE); +SELECT PARTITION_TABLE('store.store_orders_fact'); +CREATE PROJECTION ytd_orders AS SELECT * FROM store.store_orders_fact ORDER BY date_ordered + ON PARTITION RANGE BETWEEN date_trunc('year',now())::date AND NULL; +SELECT start_refresh(); \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/vertica/docker-compose.yml b/metadata-ingestion/tests/integration/vertica/docker-compose.yml index 1b22d804b0a37..ddaf206f236cf 100644 --- a/metadata-ingestion/tests/integration/vertica/docker-compose.yml +++ b/metadata-ingestion/tests/integration/vertica/docker-compose.yml @@ -3,7 +3,7 @@ services: vertica: environment: APP_DB_USER: "dbadmin" - APP_DB_PASSWORD: "vertica" + APP_DB_PASSWORD: "abc123" container_name: vertica-ce image: vertica/vertica-ce:12.0.2-0 ports: @@ -20,3 +20,4 @@ volumes: vertica-data: + diff --git a/metadata-ingestion/tests/integration/vertica/test_vertica.py b/metadata-ingestion/tests/integration/vertica/test_vertica.py index 384dac9b8d0c0..db8bfd247313b 100644 --- a/metadata-ingestion/tests/integration/vertica/test_vertica.py +++ b/metadata-ingestion/tests/integration/vertica/test_vertica.py @@ -1,24 +1,34 @@ +import subprocess +import time +from typing import List, Optional + import pytest from freezegun import freeze_time from tests.test_helpers import mce_helpers from tests.test_helpers.click_helpers import run_datahub_cmd -from tests.test_helpers.docker_helpers import is_responsive, wait_for_port +from tests.test_helpers.docker_helpers import wait_for_port FROZEN_TIME = "2020-04-14 07:00:00" -pytestmark = pytest.mark.skip( - reason="Vertica tests are disabled due to a dependency conflict with SQLAlchemy 1.3.24" -) - @pytest.fixture(scope="module") def test_resources_dir(pytestconfig): return pytestconfig.rootpath / "tests/integration/vertica" +def is_vertica_responsive( + container_name: str, port: int, hostname: Optional[str] +) -> bool: + if hostname: + cmd = f"docker logs {container_name} 2>&1 | grep 'Vertica is now running' " + ret = subprocess.run(cmd, shell=True, stdout=subprocess.DEVNULL) + + return ret.returncode == 0 + + @pytest.fixture(scope="module") -def vertica_runner(docker_compose_runner, pytestconfig, test_resources_dir): +def vertica_runner(docker_compose_runner, test_resources_dir): with docker_compose_runner( test_resources_dir / "docker-compose.yml", "vertica" ) as docker_services: @@ -27,26 +37,49 @@ def vertica_runner(docker_compose_runner, pytestconfig, test_resources_dir): "vertica-ce", 5433, timeout=120, - checker=lambda: is_responsive("vertica-ce", 5433, hostname="vertica-ce"), + checker=lambda: is_vertica_responsive( + "vertica-ce", 5433, hostname="vertica-ce" + ), ) + + commands = """ + docker cp tests/integration/vertica/ddl.sql vertica-ce:/home/dbadmin/ && + docker exec vertica-ce sh -c "/opt/vertica/bin/vsql -w abc123 -f /home/dbadmin/ddl.sql + """ + + ret = subprocess.run(commands, shell=True, stdout=subprocess.DEVNULL) + # waiting for vertica to create default table and system table and ml models + time.sleep(60) + + assert ret.returncode >= 1 + yield docker_services # Test needs more work to be done , currently it is working fine. @freeze_time(FROZEN_TIME) @pytest.mark.integration -@pytest.mark.skip("This does not work yet and needs to be fixed.") def test_vertica_ingest_with_db(vertica_runner, pytestconfig, tmp_path): test_resources_dir = pytestconfig.rootpath / "tests/integration/vertica" + # Run the metadata ingestion pipeline. config_file = (test_resources_dir / "vertica_to_file.yml").resolve() run_datahub_cmd( ["ingest", "--strict-warnings", "-c", f"{config_file}"], tmp_path=tmp_path ) + ignore_paths: List[str] = [ + r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['create_time'\]", + r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['table_size'\]", + r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['projection_size'\]", + r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['ROS_Count'\]", + r"root\[\d+\]\['aspect'\].+\['customProperties'\]\['cluster_size'\]", + r"root\[\d+\]\['aspect'\].+\['customProperties'\]\['udx_language'\]", + ] # Verify the output. mce_helpers.check_golden_file( pytestconfig, + ignore_paths=ignore_paths, output_path=tmp_path / "vertica.json", golden_path=test_resources_dir / "vertica_mces_with_db_golden.json", ) diff --git a/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json b/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json index 7c2de60372b67..44a5e07d7b996 100644 --- a/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json +++ b/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json @@ -8,10 +8,10 @@ "json": { "customProperties": { "platform": "vertica", - "instance": "PROD", + "env": "PROD", "database": "vmart", "cluster_type": "Enterprise", - "cluster_size": "87 GB", + "cluster_size": "122 GB", "subcluster": " ", "communal_storage_path": "" }, @@ -70,6 +70,21 @@ "runId": "vertica-2020_04_14-07_00_00" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", @@ -79,12 +94,12 @@ "json": { "customProperties": { "platform": "vertica", - "instance": "PROD", + "env": "PROD", "database": "vmart", "schema": "public", - "projection_count": "0", - "udx_list": "", - "Udx_langauge": "" + "projection_count": "9", + "udx_list": "APPROXIMATE_COUNT_DISTINCT_SYNOPSIS_INFO, APPROXIMATE_MEDIAN, APPROXIMATE_PERCENTILE, AcdDataToCount, AcdDataToLongSyn, AcdDataToSyn, AcdSynToCount, AcdSynToSyn, DelimitedExport, DelimitedExportMulti, EmptyMap, Explode, FAvroParser, FCefParser, FCsvParser, FDelimitedPairParser, FDelimitedParser, FIDXParser, FJSONParser, FRegexParser, FlexTokenizer, JsonExport, JsonExportMulti, KafkaAvroParser, KafkaCheckBrokers, KafkaExport, KafkaInsertDelimiters, KafkaInsertLengths, KafkaJsonParser, KafkaListManyTopics, KafkaListTopics, KafkaOffsets, KafkaParser, KafkaSource, KafkaTopicDetails, MSE, MapAggregate, MapAggregate, MapContainsKey, MapContainsKey, MapContainsValue, MapContainsValue, MapDelimitedExtractor, MapItems, MapItems, MapJSONExtractor, MapKeys, MapKeys, MapKeysInfo, MapKeysInfo, MapLookup, MapLookup, MapLookup, MapPut, MapRegexExtractor, MapSize, MapSize, MapToString, MapToString, MapValues, MapValues, MapValuesOrField, MapVersion, MapVersion, OrcExport, OrcExportMulti, PRC, ParquetExport, ParquetExportMulti, PickBestType, PickBestType, PickBestType, ROC, STV_AsGeoJSON, STV_AsGeoJSON, STV_AsGeoJSON, STV_Create_Index, STV_Create_Index, STV_Create_Index, STV_DWithin, STV_DWithin, STV_DWithin, STV_Describe_Index, STV_Drop_Index, STV_Export2Shapefile, STV_Extent, STV_Extent, STV_ForceLHR, STV_Geography, STV_Geography, STV_GeographyPoint, STV_Geometry, STV_Geometry, STV_GeometryPoint, STV_GeometryPoint, STV_GetExportShapefileDirectory, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_IsValidReason, STV_IsValidReason, STV_IsValidReason, STV_LineStringPoint, STV_LineStringPoint, STV_LineStringPoint, STV_MemSize, STV_MemSize, STV_MemSize, STV_NN, STV_NN, STV_NN, STV_PolygonPoint, STV_PolygonPoint, STV_PolygonPoint, STV_Refresh_Index, STV_Refresh_Index, STV_Refresh_Index, STV_Rename_Index, STV_Reverse, STV_SetExportShapefileDirectory, STV_ShpCreateTable, STV_ShpParser, STV_ShpSource, ST_Area, ST_Area, ST_Area, ST_AsBinary, ST_AsBinary, ST_AsBinary, ST_AsText, ST_AsText, ST_AsText, ST_Boundary, ST_Buffer, ST_Centroid, ST_Contains, ST_Contains, ST_Contains, ST_ConvexHull, ST_Crosses, ST_Difference, ST_Disjoint, ST_Disjoint, ST_Disjoint, ST_Distance, ST_Distance, ST_Distance, ST_Envelope, ST_Equals, ST_Equals, ST_Equals, ST_GeoHash, ST_GeoHash, ST_GeoHash, ST_GeographyFromText, ST_GeographyFromWKB, ST_GeomFromGeoHash, ST_GeomFromGeoJSON, ST_GeomFromGeoJSON, ST_GeomFromText, ST_GeomFromText, ST_GeomFromWKB, ST_GeomFromWKB, ST_GeometryN, ST_GeometryN, ST_GeometryN, ST_GeometryType, ST_GeometryType, ST_GeometryType, ST_Intersection, ST_Intersects, ST_Intersects, ST_IsEmpty, ST_IsEmpty, ST_IsEmpty, ST_IsSimple, ST_IsSimple, ST_IsSimple, ST_IsValid, ST_IsValid, ST_IsValid, ST_Length, ST_Length, ST_Length, ST_NumGeometries, ST_NumGeometries, ST_NumGeometries, ST_NumPoints, ST_NumPoints, ST_NumPoints, ST_Overlaps, ST_PointFromGeoHash, ST_PointN, ST_PointN, ST_PointN, ST_Relate, ST_SRID, ST_SRID, ST_SRID, ST_Simplify, ST_SimplifyPreserveTopology, ST_SymDifference, ST_Touches, ST_Touches, ST_Touches, ST_Transform, ST_Union, ST_Union, ST_Within, ST_Within, ST_Within, ST_X, ST_X, ST_X, ST_XMax, ST_XMax, ST_XMax, ST_XMin, ST_XMin, ST_XMin, ST_Y, ST_Y, ST_Y, ST_YMax, ST_YMax, ST_YMax, ST_YMin, ST_YMin, ST_YMin, ST_intersects, SetMapKeys, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_NumCol, VoltageSecureAccess, VoltageSecureAccess, VoltageSecureConfigure, VoltageSecureConfigureGlobal, VoltageSecureProtect, VoltageSecureProtect, VoltageSecureProtectAllKeys, VoltageSecureRefreshPolicy, VoltageSecureVersion, append_centers, apply_bisecting_kmeans, apply_iforest, apply_inverse_pca, apply_inverse_svd, apply_kmeans, apply_normalize, apply_one_hot_encoder, apply_pca, apply_svd, approximate_quantiles, ar_create_blobs, ar_final_newton, ar_save_model, ar_transition_newton, avg_all_columns_local, bisecting_kmeans_init_model, bk_apply_best_kmeans_results, bk_compute_totss_local, bk_finalize_model, bk_get_rows_in_active_cluster, bk_kmeans_compute_local_centers, bk_kmeans_compute_withinss, bk_kmeans_fast_random_init, bk_kmeans_slow_random_init, bk_kmeanspp_init_cur_cluster, bk_kmeanspp_reset_blob, bk_kmeanspp_select_new_centers, bk_kmeanspp_within_chunk_sum, bk_save_final_model, bk_write_new_cluster_level, blob_to_table, bufUdx, bufUdx, calc_pseudo_centers, calculate_alpha_linear, calculate_hessian_linear1, calculate_hessian_linear2, cleanup_kmeans_files, compute_and_save_global_center, compute_and_save_new_centers, compute_local_totss, compute_local_withinss, compute_new_local_centers, confusion_matrix, coordinate_descent_covariance, corr_matrix, count_rows_in_blob, create_aggregator_blob, error_rate, evaluate_naive_bayes_model, evaluate_reg_model, evaluate_svm_model, export_model_files, finalize_blob_resource_group, get_attr_minmax, get_attr_robust_zscore, get_attr_zscore, get_model_attribute, get_model_summary, get_robust_zscore_median, iforest_create_blobs, iforest_phase0_udf1, iforest_phase0_udf2, iforest_phase1_udf1, iforest_phase1_udf2, iforest_phase1_udf3, iforest_phase1_udf4, iforest_phase2_udf1, iforest_phase2_udf2, iforest_phase2_udf3, iforest_phase2_udf4, iforest_save_model, import_model_files, isOrContains, kmeansAddMetricsToModel, kmeans_init_blobs, kmeans_to_write_final_centers, lift_table, line_search_logistic1, line_search_logistic2, load_rows_into_blocks, map_factor, math_op, matrix_global_xtx, matrix_local_xtx, mode_finder, model_converter, naive_bayes_phase1, naive_bayes_phase1_blob, naive_bayes_phase2, pca_prep1_global, pca_prep1_local, pca_prep2, pmml_parser, predict_autoregressor, predict_linear_reg, predict_logistic_reg, predict_moving_average, predict_naive_bayes, predict_naive_bayes_classes, predict_pmml, predict_rf_classifier, predict_rf_classifier_classes, predict_rf_regressor, predict_svm_classifier, predict_svm_regressor, predict_xgb_classifier, predict_xgb_classifier_classes, predict_xgb_regressor, random_init, random_init_write, read_from_dfblob, read_map_factor, read_ptree, read_tree, reg_final_bfgs, reg_final_newton, reg_transition_bfgs, reg_transition_newton, reg_write_model, remove_blob, reverse_normalize, rf_blob, rf_clean, rf_phase0_udf1, rf_phase0_udf2, rf_phase1_udf1, rf_phase1_udf2, rf_phase1_udf3, rf_phase1_udf4, rf_phase2_udf1, rf_phase2_udf2, rf_phase2_udf3, rf_phase2_udf4, rf_predictor_importance, rf_save_model, rsquared, save_cv_result, save_pca_model, save_svd_model, save_svm_model, select_new_centers, store_minmax_model, store_one_hot_encoder_model, store_robust_zscore_model, store_zscore_model, table_to_blob, table_to_dfblob, update_and_return_sum_of_squared_distances, upgrade_model_format, writeInitialKmeansModelToDfs, xgb_create_blobs, xgb_phase0_udf1, xgb_phase0_udf2, xgb_phase1_udf1, xgb_phase1_udf2, xgb_phase1_udf3, xgb_phase2_udf1, xgb_phase2_udf2, xgb_phase2_udf3, xgb_prune, xgb_save_model, yule_walker, ", + "udx_language": "ComplexTypesLib -- Functions for Complex Types | DelimitedExportLib -- Delimited data export package | JsonExportLib -- Json data export package | MachineLearningLib -- Machine learning package | OrcExportLib -- Orc export package | ParquetExportLib -- Parquet export package | ApproximateLib -- Approximate package | FlexTableLib -- Flexible Tables Data Load and Query | KafkaLib -- Kafka streaming load and export | PlaceLib -- Geospatial package | VoltageSecureLib -- Voltage SecureData Connector | " }, "name": "public" } @@ -155,5 +170,10210 @@ "lastObserved": 1586847600000, "runId": "vertica-2020_04_14-07_00_00" } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_time": "2023-07-26 06:37:53.358215+00:00" + }, + "name": "customer_dimension", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "public.customer_dimension", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "customer_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=16)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=256)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_gender", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=8)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "title", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=8)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "household_id", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_address", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=256)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_city", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_state", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_region", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "marital_status", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_age", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number_of_children", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "annual_income", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "occupation", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "largest_bill_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_membership_card", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_since", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "deal_stage", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "deal_size", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_deal_update", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_time": "2023-07-26 06:37:53.368954+00:00" + }, + "name": "date_dimension", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "public.date_dimension", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "date_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "full_date_description", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=18)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "day_of_week", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=9)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "day_number_in_calendar_month", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "day_number_in_calendar_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "day_number_in_fiscal_month", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "day_number_in_fiscal_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_day_in_week_indicator", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_day_in_month_indicator", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_week_number_in_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_month_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=9)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_month_number_in_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_year_month", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=7)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_quarter", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_year_quarter", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=7)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_half_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "holiday_indicator", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=10)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weekday_indicator", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=7)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "selling_season", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_time": "2023-07-26 06:37:53.375896+00:00" + }, + "name": "employee_dimension", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "public.employee_dimension", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "employee_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "employee_gender", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=8)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "courtesy_title", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=8)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_first_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_middle_initial", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=8)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_last_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_age", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "hire_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_street_address", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=256)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_city", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_state", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_region", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "job_title", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reports_to", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "salaried_flag", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "annual_salary", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "hourly_rate", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vacation_days", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_time": "2023-07-26 06:37:53.385843+00:00" + }, + "name": "inventory_fact", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "public.inventory_fact", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "date_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_version", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warehouse_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "qty_in_stock", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "inventory_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_time": "2023-07-26 06:37:53.362016+00:00" + }, + "name": "product_dimension", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "public.product_dimension", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "product_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "product_version", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_description", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=128)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sku_number", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category_description", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "department_description", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "package_type_description", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "package_size", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fat_content", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "diet_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weight", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weight_units_of_measure", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shelf_width", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shelf_height", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shelf_depth", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_price", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_cost", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "lowest_competitor_price", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "highest_competitor_price", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "average_competitor_price", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "discontinued_flag", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_time": "2023-07-26 06:37:53.365453+00:00" + }, + "name": "promotion_dimension", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "public.promotion_dimension", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "promotion_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "promotion_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=128)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price_reduction_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "promotion_media_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ad_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "display_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "coupon_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ad_media_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "display_provider", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=128)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "promotion_cost", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "promotion_begin_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "promotion_end_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_time": "2023-07-26 06:37:53.379273+00:00" + }, + "name": "shipping_dimension", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "public.shipping_dimension", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "shipping_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "ship_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=30)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ship_mode", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=10)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ship_carrier", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=20)", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_time": "2023-07-26 06:37:53.372409+00:00" + }, + "name": "vendor_dimension", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "public.vendor_dimension", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "vendor_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "vendor_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor_address", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor_city", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor_state", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor_region", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "deal_size", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_deal_update", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vmart_load_success,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vmart_load_success,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vmart_load_success,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_time": "2023-07-26 06:38:20.045598+00:00" + }, + "name": "vmart_load_success", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "public.vmart_load_success", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "a", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vmart_load_success,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vmart_load_success,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_time": "2023-07-26 06:37:53.382549+00:00" + }, + "name": "warehouse_dimension", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "public.warehouse_dimension", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "warehouse_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "warehouse_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=20)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warehouse_address", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=256)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warehouse_city", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=60)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warehouse_state", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warehouse_region", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "is_segmented": "True", + "Segmentation_key": "hash(date_dimension.date_key)", + "projection_size": "138 KB", + "Partition_Key": "Not Available", + "Partition_Size": "0", + "Projection_Cached": "False" + }, + "name": "date_dimension_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "public.date_dimension_super", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "date_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "full_date_description", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=18)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "day_of_week", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=9)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "day_number_in_calendar_month", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "day_number_in_calendar_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "day_number_in_fiscal_month", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "day_number_in_fiscal_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_day_in_week_indicator", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_day_in_month_indicator", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_week_number_in_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_month_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=9)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_month_number_in_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_year_month", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=7)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_quarter", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_year_quarter", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=7)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_half_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_year", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "holiday_indicator", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=10)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weekday_indicator", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=7)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "selling_season", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Projections" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.date_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "is_segmented": "True", + "Segmentation_key": "hash(product_dimension.product_key, product_dimension.product_version)", + "projection_size": "19 KB", + "Partition_Key": "Not Available", + "Partition_Size": "0", + "Projection_Cached": "False" + }, + "name": "product_dimension_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "public.product_dimension_super", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "product_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_version", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_description", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=128)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sku_number", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category_description", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "department_description", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "package_type_description", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "package_size", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fat_content", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "diet_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weight", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weight_units_of_measure", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shelf_width", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shelf_height", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shelf_depth", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_price", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_cost", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "lowest_competitor_price", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "highest_competitor_price", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "average_competitor_price", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "discontinued_flag", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Projections" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.product_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "is_segmented": "True", + "Segmentation_key": "hash(promotion_dimension.promotion_key)", + "projection_size": "3 KB", + "Partition_Key": "Not Available", + "Partition_Size": "0", + "Projection_Cached": "False" + }, + "name": "promotion_dimension_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "public.promotion_dimension_super", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "promotion_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "promotion_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=128)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price_reduction_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "promotion_media_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ad_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "display_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "coupon_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ad_media_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "display_provider", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=128)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "promotion_cost", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "promotion_begin_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "promotion_end_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Projections" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.promotion_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "is_segmented": "True", + "Segmentation_key": "hash(vendor_dimension.vendor_key)", + "projection_size": "1 KB", + "Partition_Key": "Not Available", + "Partition_Size": "0", + "Projection_Cached": "False" + }, + "name": "vendor_dimension_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "public.vendor_dimension_super", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "vendor_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor_address", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor_city", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor_state", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor_region", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "deal_size", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_deal_update", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Projections" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.vendor_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "is_segmented": "True", + "Segmentation_key": "hash(customer_dimension.customer_key)", + "projection_size": "2119 KB", + "Partition_Key": "Not Available", + "Partition_Size": "0", + "Projection_Cached": "False" + }, + "name": "customer_dimension_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "public.customer_dimension_super", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=16)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=256)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_gender", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=8)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "title", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=8)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "household_id", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_address", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=256)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_city", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_state", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_region", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "marital_status", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_age", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number_of_children", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "annual_income", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "occupation", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "largest_bill_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_membership_card", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_since", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "deal_stage", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "deal_size", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_deal_update", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Projections" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.customer_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "is_segmented": "True", + "Segmentation_key": "hash(employee_dimension.employee_key)", + "projection_size": "327 KB", + "Partition_Key": "Not Available", + "Partition_Size": "0", + "Projection_Cached": "False" + }, + "name": "employee_dimension_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "public.employee_dimension_super", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "employee_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_gender", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=8)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "courtesy_title", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=8)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_first_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_middle_initial", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=8)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_last_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_age", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "hire_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_street_address", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=256)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_city", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_state", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_region", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "job_title", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reports_to", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "salaried_flag", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "annual_salary", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "hourly_rate", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vacation_days", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Projections" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.employee_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "is_segmented": "True", + "Segmentation_key": "hash(warehouse_dimension.warehouse_key)", + "projection_size": "2 KB", + "Partition_Key": "Not Available", + "Partition_Size": "0", + "Projection_Cached": "False" + }, + "name": "warehouse_dimension_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "public.warehouse_dimension_super", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "warehouse_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warehouse_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=20)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warehouse_address", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=256)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warehouse_city", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=60)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warehouse_state", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warehouse_region", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Projections" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.warehouse_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "is_segmented": "True", + "Segmentation_key": "hash(shipping_dimension.shipping_key)", + "projection_size": "1 KB", + "Partition_Key": "Not Available", + "Partition_Size": "0", + "Projection_Cached": "False" + }, + "name": "shipping_dimension_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "public.shipping_dimension_super", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "shipping_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ship_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=30)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ship_mode", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=10)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ship_carrier", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=20)", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Projections" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.shipping_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "is_segmented": "True", + "Segmentation_key": "hash(inventory_fact.date_key, inventory_fact.product_key, inventory_fact.product_version, inventory_fact.warehouse_key, inventory_fact.qty_in_stock)", + "projection_size": "2566 KB", + "Partition_Key": "Not Available", + "Partition_Size": "0", + "Projection_Cached": "False" + }, + "name": "inventory_fact_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "public.inventory_fact_super", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "date_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_version", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warehouse_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "qty_in_stock", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "inventory_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Projections" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,public.inventory_fact_super,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4", + "urn": "urn:li:container:eb682025a9113b5543ec7ed26bfa21e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "vertica", + "env": "PROD", + "database": "vmart", + "schema": "store", + "projection_count": "3", + "udx_list": "", + "udx_language": "" + }, + "name": "store" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:vertica" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:343f520ad0fb3259b298736800bb1385" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_time": "2023-07-26 06:37:53.393181+00:00" + }, + "name": "store_dimension", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "store.store_dimension", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "store_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "store_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_number", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_address", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=256)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_city", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_state", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_region", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "floor_plan_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "photo_processing_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "financial_service_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "selling_square_footage", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "total_square_footage", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_open_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_remodel_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number_of_employees", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "annual_shrinkage", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "foot_traffic", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "monthly_rent_cost", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_time": "2023-07-26 06:37:53.404717+00:00" + }, + "name": "store_orders_fact", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "store.store_orders_fact", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "product_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_version", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order_number", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "date_ordered", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "date_shipped", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_delivery_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "date_delivered", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "quantity_ordered", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "quantity_delivered", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shipper_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "unit_price", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shipping_cost", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "total_order_cost", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "quantity_in_stock", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reorder_level", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "overstock_ceiling", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_time": "2023-07-26 06:37:53.396731+00:00" + }, + "name": "store_sales_fact", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "store.store_sales_fact", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "date_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_version", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "promotion_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pos_transaction_number", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sales_quantity", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sales_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "gross_profit_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "transaction_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=16)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "transaction_time", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "TIME()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "tender_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=8)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_sales_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_sales_datetime", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "TIMESTAMP_WITH_PRECISION()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension_super,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "is_segmented": "True", + "Segmentation_key": "hash(store_dimension.store_key)", + "projection_size": "2 KB", + "Partition_Key": "Not Available", + "Partition_Size": "0", + "Projection_Cached": "False" + }, + "name": "store_dimension_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "store.store_dimension_super", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "store_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_number", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_address", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=256)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_city", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_state", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_region", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "floor_plan_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "photo_processing_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "financial_service_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "selling_square_footage", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "total_square_footage", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_open_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_remodel_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number_of_employees", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "annual_shrinkage", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "foot_traffic", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "monthly_rent_cost", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Projections" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact_super,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact_super,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact_super,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "ROS_Count": "2", + "Projection_Type": "is_super_projection", + "is_segmented": "True", + "Segmentation_key": "hash(store_sales_fact.date_key, store_sales_fact.product_key, store_sales_fact.product_version, store_sales_fact.store_key, store_sales_fact.promotion_key, store_sales_fact.customer_key, store_sales_fact.employee_key, store_sales_fact.pos_transaction_number)", + "projection_size": "225089 KB", + "Partition_Key": "Not Available", + "Partition_Size": "0", + "Projection_Cached": "False" + }, + "name": "store_sales_fact_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "store.store_sales_fact_super", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "date_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_version", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "promotion_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pos_transaction_number", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sales_quantity", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sales_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "gross_profit_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "transaction_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=16)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "transaction_time", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "TIME()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "tender_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=8)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_sales_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_sales_datetime", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "TIMESTAMP_WITH_PRECISION()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact_super,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Projections" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact_super,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_sales_fact_super,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact_super,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact_super,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:342b43fc61f85b16580be55c11e89787" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact_super,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "is_segmented": "True", + "Segmentation_key": "hash(store_orders_fact.product_key, store_orders_fact.product_version, store_orders_fact.store_key, store_orders_fact.vendor_key, store_orders_fact.employee_key, store_orders_fact.order_number, store_orders_fact.date_ordered, store_orders_fact.date_shipped)", + "projection_size": "8648 KB", + "Partition_Key": "Not Available", + "Partition_Size": "0", + "Projection_Cached": "False" + }, + "name": "store_orders_fact_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "store.store_orders_fact_super", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "product_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_version", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "employee_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order_number", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "date_ordered", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "date_shipped", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_delivery_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "date_delivered", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "quantity_ordered", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "quantity_delivered", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shipper_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "unit_price", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shipping_cost", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "total_order_cost", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "quantity_in_stock", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reorder_level", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "overstock_ceiling", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact_super,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Projections" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact_super,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,store.store_orders_fact_super,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:342b43fc61f85b16580be55c11e89787", + "urn": "urn:li:container:342b43fc61f85b16580be55c11e89787" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "vertica", + "env": "PROD", + "database": "vmart", + "schema": "online_sales", + "projection_count": "3", + "udx_list": "", + "udx_language": "" + }, + "name": "online_sales" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:vertica" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:343f520ad0fb3259b298736800bb1385" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_time": "2023-07-26 06:37:53.415595+00:00" + }, + "name": "call_center_dimension", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "online_sales.call_center_dimension", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "call_center_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "cc_closed_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_open_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=50)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_class", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=50)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_employees", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_hours", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=20)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_manager", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=40)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_address", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=256)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_city", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_state", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_region", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "urn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_time": "2023-07-26 06:37:53.412266+00:00" + }, + "name": "online_page_dimension", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "online_sales.online_page_dimension", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "online_page_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "start_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "end_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "page_number", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "page_description", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=100)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "page_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=100)", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "urn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_time": "2023-07-26 06:37:53.419260+00:00" + }, + "name": "online_sales_fact", + "description": "References the properties of a native table in Vertica. Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution. In order to query or perform any operation on a Vertica table, the table must have one or more projections associated with it. ", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "online_sales.online_sales_fact", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sale_date_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ship_date_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_version", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "call_center_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "online_page_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shipping_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warehouse_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "promotion_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pos_transaction_number", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sales_quantity", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sales_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ship_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "net_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "gross_profit_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "transaction_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=16)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "online_sales_saledate", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "online_sales_shipdate", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "urn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension_super,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "is_segmented": "True", + "Segmentation_key": "hash(online_page_dimension.online_page_key)", + "projection_size": "9 KB", + "Partition_Key": "Not Available", + "Partition_Size": "0", + "Projection_Cached": "False" + }, + "name": "online_page_dimension_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "online_sales.online_page_dimension_super", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "online_page_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "start_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "end_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "page_number", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "page_description", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=100)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "page_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=100)", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Projections" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_page_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "urn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension_super,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "is_segmented": "True", + "Segmentation_key": "hash(call_center_dimension.call_center_key)", + "projection_size": "6 KB", + "Partition_Key": "Not Available", + "Partition_Size": "0", + "Projection_Cached": "False" + }, + "name": "call_center_dimension_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "online_sales.call_center_dimension_super", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "call_center_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_closed_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_open_date", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_name", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=50)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_class", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=50)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_employees", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_hours", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=20)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_manager", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=40)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_address", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=256)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_city", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_state", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "CHAR(length=2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cc_region", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=64)", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Projections" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.call_center_dimension_super,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "urn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact_super,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:dbadmin", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact_super,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact_super,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "ROS_Count": "1", + "Projection_Type": "is_super_projection", + "is_segmented": "True", + "Segmentation_key": "hash(online_sales_fact.sale_date_key, online_sales_fact.ship_date_key, online_sales_fact.product_key, online_sales_fact.product_version, online_sales_fact.customer_key, online_sales_fact.call_center_key, online_sales_fact.online_page_key, online_sales_fact.shipping_key)", + "projection_size": "182385 KB", + "Partition_Key": "Not Available", + "Partition_Size": "0", + "Projection_Cached": "False" + }, + "name": "online_sales_fact_super", + "description": "Vertica physically stores table data in projections, which are collections of table columns. Projections store data in a format that optimizes query execution For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "online_sales.online_sales_fact_super", + "platform": "urn:li:dataPlatform:vertica", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sale_date_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ship_date_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "product_version", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "call_center_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "online_page_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shipping_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warehouse_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "promotion_key", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pos_transaction_number", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sales_quantity", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sales_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ship_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "net_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "gross_profit_dollar_amount", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "transaction_type", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=16)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "online_sales_saledate", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "online_sales_shipdate", + "nullable": true, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact_super,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Projections" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact_super,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:vertica,online_sales.online_sales_fact_super,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:343f520ad0fb3259b298736800bb1385", + "urn": "urn:li:container:343f520ad0fb3259b298736800bb1385" + }, + { + "id": "urn:li:container:ae8df3182db1bb8b3a612998126beae7", + "urn": "urn:li:container:ae8df3182db1bb8b3a612998126beae7" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "vertica-2020_04_14-07_00_00" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml b/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml index da1fcc2b5304f..ebd800ee09ff5 100644 --- a/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml +++ b/metadata-ingestion/tests/integration/vertica/vertica_to_file.yml @@ -4,7 +4,7 @@ source: host_port: 127.0.0.1:5433 database: Vmart username: dbadmin - password: vertica + password: abc123 sink: type: file From 66074341f751ee825c623f7625bf5bd6d8de10a2 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 1 Aug 2023 07:04:48 -0700 Subject: [PATCH 214/222] test(ingest): test case statements with sql parser (#8437) --- .../test_snowflake_case_statement.json | 37 +++++++++++++++++++ .../unit/sql_parsing/test_sqlglot_lineage.py | 28 ++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_case_statement.json diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_case_statement.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_case_statement.json new file mode 100644 index 0000000000000..7d1a4f2039b10 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_snowflake_case_statement.json @@ -0,0 +1,37 @@ +{ + "query_type": "SELECT", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)" + ], + "out_tables": [], + "column_lineage": [ + { + "downstream": { + "table": null, + "column": "TOTAL_PRICE_CATEGORY" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)", + "column": "totalprice" + } + ] + }, + { + "downstream": { + "table": null, + "column": "TOTAL_PRICE_SUCCESS" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)", + "column": "is_payment_successful" + }, + { + "table": "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)", + "column": "totalprice" + } + ] + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py index 0d8471291a505..5a294be150fa0 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py +++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py @@ -287,6 +287,34 @@ def test_snowflake_column_normalization(): ) +def test_snowflake_case_statement(): + assert_sql_result( + """ +SELECT + CASE + WHEN o."totalprice" > 1000 THEN 'high' + WHEN o."totalprice" > 100 THEN 'medium' + ELSE 'low' + END as total_price_category, + -- Also add a case where the column is in the THEN clause. + CASE + WHEN o."is_payment_successful" THEN o."totalprice" + ELSE 0 + END as total_price_success +FROM snowflake_sample_data.tpch_sf1.orders o +""", + dialect="snowflake", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,snowflake_sample_data.tpch_sf1.orders,PROD)": { + "orderkey": "NUMBER", + "totalprice": "FLOAT", + "is_payment_successful": "BOOLEAN", + }, + }, + expected_file=RESOURCE_DIR / "test_snowflake_case_statement.json", + ) + + @pytest.mark.skip(reason="We don't handle the unnest lineage correctly") def test_bigquery_unnest_columns(): assert_sql_result( From 547e1f498e37ee7ac7921cf1fe552f1e63d3b2e1 Mon Sep 17 00:00:00 2001 From: mohdsiddique Date: Tue, 1 Aug 2023 19:35:18 +0530 Subject: [PATCH 215/222] feat(ingestion/tableau): support column level lineage for custom sql (#8466) Co-authored-by: MohdSiddiqueBagwan --- .../src/datahub/ingestion/source/tableau.py | 274 +- .../ingestion/source/tableau_common.py | 103 +- .../setup/customSQLTablesConnection_all.json | 64 +- .../embeddedDatasourcesConnection_all.json | 2 +- .../tableau/tableau_cll_mces_golden.json | 43283 ++++++++++++++++ ...bleau_extract_all_project_mces_golden.json | 151 +- .../tableau/tableau_mces_golden.json | 151 +- .../tableau_mces_golden_deleted_stateful.json | 15 + .../tableau_nested_project_mces_golden.json | 122 +- .../tableau_signout_timeout_mces_golden.json | 151 +- ...au_with_platform_instance_mces_golden.json | 156 +- .../tableau/test_tableau_ingest.py | 101 +- 12 files changed, 44478 insertions(+), 95 deletions(-) create mode 100644 metadata-ingestion/tests/integration/tableau/tableau_cll_mces_golden.json diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau.py index e923727a990f4..6752bdf519830 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau.py @@ -4,14 +4,24 @@ from dataclasses import dataclass from datetime import datetime from functools import lru_cache -from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union, cast +from typing import ( + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Set, + Tuple, + Union, + cast, +) import dateutil.parser as dp import tableauserverclient as TSC from pydantic import root_validator, validator from pydantic.fields import Field from requests.adapters import ConnectionError -from sqllineage.runner import LineageRunner from tableauserverclient import ( PersonalAccessTokenAuth, Server, @@ -71,8 +81,11 @@ dashboard_graphql_query, database_tables_graphql_query, embedded_datasource_graphql_query, + get_overridden_info, get_unique_custom_sql, + make_fine_grained_lineage_class, make_table_urn, + make_upstream_class, published_datasource_graphql_query, query_metadata, sheet_graphql_query, @@ -120,10 +133,15 @@ OwnershipClass, OwnershipTypeClass, SubTypesClass, - UpstreamClass, ViewPropertiesClass, ) from datahub.utilities import config_clean +from datahub.utilities.sqlglot_lineage import ( + ColumnLineageInfo, + SchemaResolver, + SqlParsingResult, + sqlglot_lineage, +) logger: logging.Logger = logging.getLogger(__name__) @@ -870,14 +888,14 @@ def _create_upstream_table_lineage( f"A total of {len(upstream_tables)} upstream table edges found for datasource {datasource[tableau_constant.ID]}" ) - if datasource.get(tableau_constant.FIELDS): - datasource_urn = builder.make_dataset_urn_with_platform_instance( - platform=self.platform, - name=datasource[tableau_constant.ID], - platform_instance=self.config.platform_instance, - env=self.config.env, - ) + datasource_urn = builder.make_dataset_urn_with_platform_instance( + platform=self.platform, + name=datasource[tableau_constant.ID], + platform_instance=self.config.platform_instance, + env=self.config.env, + ) + if datasource.get(tableau_constant.FIELDS): if self.config.extract_column_level_lineage: # Find fine grained lineage for datasource column to datasource column edge, # upstream columns may be from same datasource @@ -1140,6 +1158,57 @@ def get_upstream_fields_of_field_in_datasource(self, datasource, datasource_urn) ) return fine_grained_lineages + def get_upstream_fields_from_custom_sql( + self, datasource: dict, datasource_urn: str + ) -> List[FineGrainedLineage]: + fine_grained_lineages: List[FineGrainedLineage] = [] + + parsed_result = self.parse_custom_sql( + datasource=datasource, + datasource_urn=datasource_urn, + env=self.config.env, + platform=self.platform, + platform_instance=self.config.platform_instance, + func_overridden_info=None, # Here we don't want to override any information from configuration + ) + + if parsed_result is None: + logger.info( + f"Failed to extract column level lineage from datasource {datasource_urn}" + ) + return fine_grained_lineages + + cll: List[ColumnLineageInfo] = ( + parsed_result.column_lineage + if parsed_result.column_lineage is not None + else [] + ) + for cll_info in cll: + downstream = ( + [ + builder.make_schema_field_urn( + datasource_urn, cll_info.downstream.column + ) + ] + if cll_info.downstream is not None + and cll_info.downstream.column is not None + else [] + ) + upstreams = [ + builder.make_schema_field_urn(column_ref.table, column_ref.column) + for column_ref in cll_info.upstreams + ] + fine_grained_lineages.append( + FineGrainedLineage( + downstreamType=FineGrainedLineageDownstreamType.FIELD, + downstreams=downstream, + upstreamType=FineGrainedLineageUpstreamType.FIELD_SET, + upstreams=upstreams, + ) + ) + + return fine_grained_lineages + def get_transform_operation(self, field): field_type = field[tableau_constant.TYPE_NAME] if field_type in ( @@ -1176,6 +1245,7 @@ def emit_custom_sql_datasources(self) -> Iterable[MetadataWorkUnit]: platform_instance=self.config.platform_instance, env=self.config.env, ) + dataset_snapshot = DatasetSnapshot( urn=csql_urn, aspects=[self.get_data_platform_instance()], @@ -1225,14 +1295,20 @@ def emit_custom_sql_datasources(self) -> Iterable[MetadataWorkUnit]: csql_urn, tables, datasource ) elif self.config.extract_lineage_from_unsupported_custom_sql_queries: + logger.debug("Extracting TLL & CLL from custom sql") # custom sql tables may contain unsupported sql, causing incomplete lineage # we extract the lineage from the raw queries yield from self._create_lineage_from_unsupported_csql( csql_urn, csql ) - # Schema Metadata - columns = csql.get(tableau_constant.COLUMNS, []) + # if condition is needed as graphQL return "cloumns": None + columns: List[Dict[Any, Any]] = ( + cast(List[Dict[Any, Any]], csql.get(tableau_constant.COLUMNS)) + if tableau_constant.COLUMNS in csql + and csql.get(tableau_constant.COLUMNS) is not None + else [] + ) schema_metadata = self.get_schema_metadata_for_custom_sql(columns) if schema_metadata is not None: dataset_snapshot.aspects.append(schema_metadata) @@ -1450,53 +1526,143 @@ def _create_lineage_to_upstream_tables( aspect=upstream_lineage, ) - def _create_lineage_from_unsupported_csql( - self, csql_urn: str, csql: dict - ) -> Iterable[MetadataWorkUnit]: - database = csql.get(tableau_constant.DATABASE) or {} + def parse_custom_sql( + self, + datasource: dict, + datasource_urn: str, + platform: str, + env: str, + platform_instance: Optional[str], + func_overridden_info: Optional[ + Callable[ + [ + str, + Optional[str], + Optional[Dict[str, str]], + Optional[TableauLineageOverrides], + ], + Tuple[Optional[str], Optional[str], str, str], + ] + ], + ) -> Optional["SqlParsingResult"]: + + database_info = datasource.get(tableau_constant.DATABASE) or {} + + if datasource.get(tableau_constant.IS_UNSUPPORTED_CUSTOM_SQL) in (None, False): + logger.debug(f"datasource {datasource_urn} is not created from custom sql") + return None + if ( - csql.get(tableau_constant.IS_UNSUPPORTED_CUSTOM_SQL, False) - and tableau_constant.NAME in database - and tableau_constant.CONNECTION_TYPE in database + tableau_constant.NAME not in database_info + or tableau_constant.CONNECTION_TYPE not in database_info ): - upstream_tables = [] - query = csql.get(tableau_constant.QUERY) - parser = LineageRunner(query) - - try: - for table in parser.source_tables: - split_table = str(table).split(".") - if len(split_table) == 2: - datset = make_table_urn( - env=self.config.env, - upstream_db=database.get(tableau_constant.NAME), - connection_type=database.get( - tableau_constant.CONNECTION_TYPE, "" - ), - schema=split_table[0], - full_name=split_table[1], - platform_instance_map=self.config.platform_instance_map, - lineage_overrides=self.config.lineage_overrides, - ) - upstream_tables.append( - UpstreamClass( - type=DatasetLineageType.TRANSFORMED, dataset=datset - ) - ) - except Exception as e: - self.report.report_warning( - key="csql-lineage", - reason=f"Unable to retrieve lineage from query. " - f"Query: {query} " - f"Reason: {str(e)} ", + logger.debug( + f"database information is missing from datasource {datasource_urn}" + ) + return None + + query = datasource.get(tableau_constant.QUERY) + if query is None: + logger.debug( + f"raw sql query is not available for datasource {datasource_urn}" + ) + return None + + logger.debug(f"Parsing sql={query}") + + upstream_db = database_info.get(tableau_constant.NAME) + + if func_overridden_info is not None: + # Override the information as per configuration + upstream_db, platform_instance, platform, _ = func_overridden_info( + database_info[tableau_constant.CONNECTION_TYPE], + database_info.get(tableau_constant.NAME), + self.config.platform_instance_map, + self.config.lineage_overrides, + ) + + logger.debug( + f"Overridden info upstream_db={upstream_db}, platform_instance={platform_instance}, platform={platform}" + ) + + parsed_result: Optional["SqlParsingResult"] = None + try: + schema_resolver = ( + self.ctx.graph._make_schema_resolver( + platform=platform, + platform_instance=platform_instance, + env=env, + ) + if self.ctx.graph is not None + else SchemaResolver( + platform=platform, + platform_instance=platform_instance, + env=env, + graph=None, ) - upstream_lineage = UpstreamLineage(upstreams=upstream_tables) - yield self.get_metadata_change_proposal( - csql_urn, - aspect_name=tableau_constant.UPSTREAM_LINEAGE, - aspect=upstream_lineage, ) + if schema_resolver.graph is None: + logger.warning( + "Column Level Lineage extraction would not work as DataHub graph client is None." + ) + + parsed_result = sqlglot_lineage( + query, + schema_resolver=schema_resolver, + default_db=upstream_db, + ) + except Exception as e: + self.report.report_warning( + key="csql-lineage", + reason=f"Unable to retrieve lineage from query. " + f"Query: {query} " + f"Reason: {str(e)} ", + ) + + return parsed_result + + def _create_lineage_from_unsupported_csql( + self, csql_urn: str, csql: dict + ) -> Iterable[MetadataWorkUnit]: + + parsed_result = self.parse_custom_sql( + datasource=csql, + datasource_urn=csql_urn, + env=self.config.env, + platform=self.platform, + platform_instance=self.config.platform_instance, + func_overridden_info=get_overridden_info, + ) + + if parsed_result is None: + logger.info( + f"Failed to extract table level lineage for datasource {csql_urn}" + ) + return + + upstream_tables = make_upstream_class(parsed_result) + + logger.debug(f"Upstream tables = {upstream_tables}") + + fine_grained_lineages: List[FineGrainedLineage] = [] + if self.config.extract_column_level_lineage: + logger.info("Extracting CLL from custom sql") + fine_grained_lineages = make_fine_grained_lineage_class( + parsed_result, csql_urn + ) + + upstream_lineage = UpstreamLineage( + upstreams=upstream_tables, + fineGrainedLineages=fine_grained_lineages, + ) + + yield self.get_metadata_change_proposal( + csql_urn, + aspect_name=tableau_constant.UPSTREAM_LINEAGE, + aspect=upstream_lineage, + ) + def _get_schema_metadata_for_datasource( self, datasource_fields: List[dict] ) -> Optional[SchemaMetadata]: diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py b/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py index e573f1d924d1a..2c92285fdba77 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py @@ -1,11 +1,17 @@ import html from functools import lru_cache -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Tuple from pydantic.fields import Field import datahub.emitter.mce_builder as builder from datahub.configuration.common import ConfigModel +from datahub.metadata.com.linkedin.pegasus2avro.dataset import ( + DatasetLineageType, + FineGrainedLineage, + FineGrainedLineageDownstreamType, + FineGrainedLineageUpstreamType, +) from datahub.metadata.com.linkedin.pegasus2avro.schema import ( ArrayTypeClass, BooleanTypeClass, @@ -21,7 +27,9 @@ BytesTypeClass, GlobalTagsClass, TagAssociationClass, + UpstreamClass, ) +from datahub.utilities.sqlglot_lineage import ColumnLineageInfo, SqlParsingResult class TableauLineageOverrides(ConfigModel): @@ -574,15 +582,13 @@ def get_platform_instance( return None -def make_table_urn( - env: str, - upstream_db: Optional[str], +def get_overridden_info( connection_type: str, - schema: str, - full_name: str, + upstream_db: Optional[str], platform_instance_map: Optional[Dict[str, str]], lineage_overrides: Optional[TableauLineageOverrides] = None, -) -> str: +) -> Tuple[Optional[str], Optional[str], str, str]: + original_platform = platform = get_platform(connection_type) if ( lineage_overrides is not None @@ -599,10 +605,38 @@ def make_table_urn( ): upstream_db = lineage_overrides.database_override_map[upstream_db] + platform_instance = get_platform_instance(original_platform, platform_instance_map) + + if original_platform in ("athena", "hive", "mysql"): # Two tier databases + upstream_db = None + + return upstream_db, platform_instance, platform, original_platform + + +def make_table_urn( + env: str, + upstream_db: Optional[str], + connection_type: str, + schema: str, + full_name: str, + platform_instance_map: Optional[Dict[str, str]], + lineage_overrides: Optional[TableauLineageOverrides] = None, +) -> str: + + upstream_db, platform_instance, platform, original_platform = get_overridden_info( + connection_type=connection_type, + upstream_db=upstream_db, + lineage_overrides=lineage_overrides, + platform_instance_map=platform_instance_map, + ) + table_name = get_fully_qualified_table_name( - original_platform, upstream_db, schema, full_name + original_platform, + upstream_db if upstream_db is not None else "", + schema, + full_name, ) - platform_instance = get_platform_instance(original_platform, platform_instance_map) + return builder.make_dataset_urn_with_platform_instance( platform, table_name, platform_instance, env ) @@ -620,6 +654,57 @@ def make_description_from_params(description, formula): return final_description +def make_upstream_class( + parsed_result: Optional[SqlParsingResult], +) -> List[UpstreamClass]: + upstream_tables: List[UpstreamClass] = [] + + if parsed_result is None: + return upstream_tables + + for dataset_urn in parsed_result.in_tables: + upstream_tables.append( + UpstreamClass(type=DatasetLineageType.TRANSFORMED, dataset=dataset_urn) + ) + return upstream_tables + + +def make_fine_grained_lineage_class( + parsed_result: Optional[SqlParsingResult], dataset_urn: str +) -> List[FineGrainedLineage]: + fine_grained_lineages: List[FineGrainedLineage] = [] + + if parsed_result is None: + return fine_grained_lineages + + cll: List[ColumnLineageInfo] = ( + parsed_result.column_lineage if parsed_result.column_lineage is not None else [] + ) + + for cll_info in cll: + downstream = ( + [builder.make_schema_field_urn(dataset_urn, cll_info.downstream.column)] + if cll_info.downstream is not None + and cll_info.downstream.column is not None + else [] + ) + upstreams = [ + builder.make_schema_field_urn(column_ref.table, column_ref.column) + for column_ref in cll_info.upstreams + ] + + fine_grained_lineages.append( + FineGrainedLineage( + downstreamType=FineGrainedLineageDownstreamType.FIELD, + downstreams=downstream, + upstreamType=FineGrainedLineageUpstreamType.FIELD_SET, + upstreams=upstreams, + ) + ) + + return fine_grained_lineages + + def get_unique_custom_sql(custom_sql_list: List[dict]) -> List[dict]: unique_custom_sql = [] for custom_sql in custom_sql_list: diff --git a/metadata-ingestion/tests/integration/tableau/setup/customSQLTablesConnection_all.json b/metadata-ingestion/tests/integration/tableau/setup/customSQLTablesConnection_all.json index 5c01adbd8a673..e0ef5a8e0b5a5 100644 --- a/metadata-ingestion/tests/integration/tableau/setup/customSQLTablesConnection_all.json +++ b/metadata-ingestion/tests/integration/tableau/setup/customSQLTablesConnection_all.json @@ -754,13 +754,73 @@ "connectionType": "postgres" } ] - } + }, + { + "id": "10c6297d-0dbd-44f1-b1ba-458bea446513", + "name": "Custom SQL Query", + "isUnsupportedCustomSql": true, + "query": "select seller_city, price from demo-custom-323403.bigquery_demo.sellers sell LEFT JOIN (\nselect * from demo-custom-323403.bigquery_demo.order_items\n) items on items.seller_id=sell.seller_id", + "columns": [ + { + "id": "2cf31227-900f-7ac8-88c9-07404c3ff832", + "name": "price", + "remoteType": "R8", + "description": null, + "referencedByFields": [ + { + "datasource": { + "__typename": "PublishedDatasource", + "id": "bda518f0-0867-679d-2390-891a05972f9e", + "name": "AbcJoinWorkbook", + "upstreamTables": [], + "projectName": "SubProject1", + "luid": "830c29b1-9d28-4e76-922e-65c0061a4b82" + } + } + ] + }, + { + "id": "d94ba54b-3c1b-410a-7109-ce0c5f82a955", + "name": "seller_city", + "remoteType": "WSTR", + "description": null, + "referencedByFields": [ + { + "datasource": { + "__typename": "PublishedDatasource", + "id": "bda518f0-0867-679d-2390-891a05972f9e", + "name": "AbcJoinWorkbook", + "upstreamTables": [], + "projectName": "SubProject1", + "luid": "830c29b1-9d28-4e76-922e-65c0061a4b82" + } + } + ] + } + ], + "tables": [], + "database": { + "name": "My Project 42084", + "connectionType": "bigquery" + }, + "datasources": [ + { + "__typename": "PublishedDatasource", + "id": "bda518f0-0867-679d-2390-891a05972f9e", + "name": "AbcJoinWorkbook", + "upstreamTables": [], + "projectName": "SubProject1", + "luid": "830c29b1-9d28-4e76-922e-65c0061a4b82" + } + ] + } ], + "pageInfo": { "hasNextPage": false, "endCursor": null }, - "totalCount": 2 + "totalCount": 3 } } } \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/setup/embeddedDatasourcesConnection_all.json b/metadata-ingestion/tests/integration/tableau/setup/embeddedDatasourcesConnection_all.json index 665076bd8affb..bdb5013c2ea48 100644 --- a/metadata-ingestion/tests/integration/tableau/setup/embeddedDatasourcesConnection_all.json +++ b/metadata-ingestion/tests/integration/tableau/setup/embeddedDatasourcesConnection_all.json @@ -1625,7 +1625,7 @@ "name": "last_name", "table": { "__typename": "CustomSQLTable", - "id": "22b0b4c3-6b85-713d-a161-5a87fdd78f40" + "id": "10c6297d-0dbd-44f1-b1ba-458bea446513" } } ] diff --git a/metadata-ingestion/tests/integration/tableau/tableau_cll_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_cll_mces_golden.json new file mode 100644 index 0000000000000..4575144c6f844 --- /dev/null +++ b/metadata-ingestion/tests/integration/tableau/tableau_cll_mces_golden.json @@ -0,0 +1,43283 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "190a6a5c-63ed-4de1-8045-faeae5df5b01" + }, + "name": "default" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "c30aafe5-44f4-4f28-80d3-d181010a263c" + }, + "name": "Project 2" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "910733aa-2e95-4ac3-a2e8-71570751099d" + }, + "name": "Samples" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "1f15d897-7f0c-7c59-037a-afa6a9b7c9a9" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15995", + "name": "Email Performance by Campaign", + "description": "Description for Email Performance by Campaign" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "661fabd0-bed6-8610-e066-0694a81a6cea" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15619", + "name": "Dvdrental Workbook", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:TagSheet3" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "6ffa5a7f-d852-78f1-6c6d-20ac23610ebf" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15605", + "name": "Executive Dashboard", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "bd040833-8f66-22c0-1b51-bd4ccf5eef7c" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/17904", + "name": "Workbook published ds", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "chartUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1638860400000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "viewsCount": 5 + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "luid": "f0779f9d-6765-47a9-a8f6-c740cfd27783" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline - Sent", + "title": "Timeline - Sent", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign List", + "title": "Campaign List", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)", + "schemaField": { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Summary", + "title": "Summary", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)", + "schemaField": { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Names)", + "schemaField": { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Values)", + "schemaField": { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile - Sent by Campaign", + "title": "Mobile - Sent by Campaign", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet1", + "title": "Sheet 1", + "description": "", + "lastModified": { + "created": { + "time": 1639772911000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)", + "schemaField": { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet2", + "title": "Sheet 2", + "description": "", + "lastModified": { + "created": { + "time": 1639773415000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)", + "schemaField": { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)", + "schemaField": { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)", + "schemaField": { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)", + "schemaField": { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)", + "schemaField": { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)", + "schemaField": { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)", + "schemaField": { + "fieldPath": "rental_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet3", + "title": "Sheet 3", + "description": "", + "lastModified": { + "created": { + "time": 1640375456000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:TagSheet3" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)", + "schemaField": { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)", + "schemaField": { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)", + "schemaField": { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Requests", + "title": "Opened Requests", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)", + "schemaField": { + "fieldPath": "Total # Request", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)", + "schemaField": { + "fieldPath": "Total Active Requests", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top 10 Items by Requests and YoY Change", + "title": "Top 10 Items by Requests and YoY Change", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Problems", + "title": "Opened Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)", + "schemaField": { + "fieldPath": "Total # Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)", + "schemaField": { + "fieldPath": "Total Active Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue", + "title": "Overdue", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High and Critical Priority Problems", + "title": "High and Critical Priority Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)", + "schemaField": { + "fieldPath": "% of critical and high priority", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total Incidents by Category and YoY Change", + "title": "Total Incidents by Category and YoY Change", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known Errors", + "title": "Known Errors", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)", + "schemaField": { + "fieldPath": "% of known error", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)", + "schemaField": { + "fieldPath": "Known error", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)", + "schemaField": { + "fieldPath": "Problems with Related Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)", + "schemaField": { + "fieldPath": "Related Incidents", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Requests", + "title": "Overdue Requests", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG Time to Solve an Incident", + "title": "AVG Time to Solve an Incident", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)", + "schemaField": { + "fieldPath": "Time to Close an Incident (seconds)", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)", + "schemaField": { + "fieldPath": "Time to Close an Incident", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made SLA?", + "title": "Made SLA?", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)", + "schemaField": { + "fieldPath": "% made SLA", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)", + "schemaField": { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-IncidentBreakdownbyPriority", + "title": "Tooltip - Incident Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Problems", + "title": "Overdue Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-ProblemBreakdownbyPriority", + "title": "Tooltip - Problem Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-RequestBreakdownbyPriority", + "title": "Tooltip - Request Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age of Active Problems", + "title": "Age of Active Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)", + "schemaField": { + "fieldPath": "Age of Problems", + "nullable": false, + "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)", + "schemaField": { + "fieldPath": "Time Span Breakdown", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Incidents", + "title": "Opened Incidents", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)", + "schemaField": { + "fieldPath": "Opened Month Tooltip", + "nullable": false, + "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)", + "schemaField": { + "fieldPath": "Total Active Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/Workbookpublishedds/Sheet1", + "title": "published sheet ds", + "description": "", + "lastModified": { + "created": { + "time": 1641951867000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642658093000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)", + "schemaField": { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)", + "schemaField": { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)", + "schemaField": { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "dashboardUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1638860400000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "viewsCount": 3 + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": { + "luid": "fc9ea488-f810-4fa8-ac19-aa96018b5d66" + }, + "title": "Email Performance by Campaign", + "description": "", + "charts": [ + "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "dvd Rental Dashboard", + "description": "", + "charts": [ + "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1639773866000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "Story 1", + "description": "", + "charts": [], + "datasets": [], + "lastModified": { + "created": { + "time": 1639773866000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "Executive Dashboard", + "description": "", + "charts": [ + "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Activity_Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Bounceback)" + ], + "transformOperation": "CalculatedFieldformula: [Sent Email] - [Delivered Email]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_Run_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign Run ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Choice_Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Choice Number %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Click Email)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)" + ], + "transformOperation": "CalculatedFieldformula: [Clickthrough Emails]/[Delivered Email]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Email Delivered)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Email Delivered)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Delivered Email]/[Sent Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Has_Predictive)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Is Predictive)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Mailing_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Mailing ID %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Non Clickthrough Email)" + ], + "transformOperation": "CalculatedFieldformula: [Delivered Email]-[Clickthrough Emails]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Opened Email]/[Delivered Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Open Email)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Non Clicked Emails)" + ], + "transformOperation": "CalculatedFieldformula: [Opened Email]-[Clickthrough Emails]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Platform)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Platform %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),updatedAt)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Updated At)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),User_Agent)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),User Agent %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),programName)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Workspace Name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-02-09T00:05:25Z", + "extractLastUpdateTime": "2018-02-09T00:05:25Z" + }, + "name": "Marketo", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Program ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Platform (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated At", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workspace Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User Agent (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Predictive", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Mobile Device", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Platform", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created At", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Mobile Device (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Device (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Link ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Non Clicked Emails", + "nullable": false, + "description": "formula: [Opened Email]-[Clickthrough Emails]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Non Clickthrough Email", + "nullable": false, + "description": "formula: [Delivered Email]-[Clickthrough Emails]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Non Opened Email", + "nullable": false, + "description": "formula: [Delivered Email]-[Opened Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User Agent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Bounceback", + "nullable": false, + "description": "formula: [Sent Email] - [Delivered Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD),last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),rental_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Customer Payment Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "actor+ (dvdrental)", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Address2", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Update", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "District", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "country, city", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Phone", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Update (Address)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Address Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Actor Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "actor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),City)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Country/Region)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Discount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Discount)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Location)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Manufacturer)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order Date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID %28Returns%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID %28Returns%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Orders)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Orders)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),People)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Postal Code)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product Name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit %28bin%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit Ratio)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Quantity)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Quantity)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region %28People%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region %28People%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Regional Manager)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Regional Manager)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returned)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returned)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returns)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returns)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Row ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Row ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sales)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Mode)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Mode)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),State)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sub-Category)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Top Customers by Profit)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Superstore Datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Region (People)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "People", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID (Returns)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Orders", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit Ratio", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returns", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Regional Manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit (bin)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Top Customers by Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T19:35:39Z", + "extractLastUpdateTime": "2018-01-18T19:35:39Z" + }, + "name": "Requests", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Closed by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mobile Picture", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% made SLA", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Meta", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total # Request", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requested for date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ordered item link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible elsewhere", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price Frequency (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Update name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Special instructions", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Execution Plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible on Bundles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No search", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "List Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer update", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Guide", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Package", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Estimated Delivery", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Billable", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price Frequency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fulfillment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Backordered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ignore price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Replace on upgrade", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Context", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requested for", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Billable (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mobile Picture Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Protection policy", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Omit price in cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Catalogs", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Entitlement script", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Published version", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Preview link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Stage (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible on Guides", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Requested Item) 1", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery time", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Stage", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Request", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolve Time", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Icon", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Display name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No proceed checkout", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Use cart layout", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No order now", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Application", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Template", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Catalog", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Image", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workflow", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Class", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created from item design", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Roles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Hide price (mobile listings)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Availability", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Vendor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Picture", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan script", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Requests", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Start closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Request state", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)" + ], + "transformOperation": "CalculatedFieldformula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T20:21:33Z", + "extractLastUpdateTime": "2018-01-18T20:21:33Z" + }, + "name": "Problems", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "SLA due (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Default assignee", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workaround", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "u", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time Span Breakdown", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "u_", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Change request", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost center", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Roles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total # Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Exclude manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lucha", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problems with Related Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Related Incidents", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Source", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of known error", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problem state", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Known error", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group email", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Age of Problems", + "nullable": false, + "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of critical and high priority", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Include members", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)" + ], + "transformOperation": "CalculatedFieldformula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T20:13:08Z", + "extractLastUpdateTime": "2018-01-18T20:13:08Z" + }, + "name": "Incidents", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Attributes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close code (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Status", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Notify (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requires verification", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Maintenance schedule", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Warranty expiration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "GL account", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First discovered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Asset", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Skip sync", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "DNS Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Caller (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Department", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolved by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Managed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "PO number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business resolve time (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Child Incidents (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "IP Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Asset tag", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due in", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Checked out", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fully qualified domain name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Installed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Purchased", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lease contract", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Vendor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent Incident (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost currency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Subcategory (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Monitor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Serial number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time to Close an Incident (seconds)", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Owned by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Invoice number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Start date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ordered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order received", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discovery source", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Class", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Operational status", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolve time (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reopen count (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Most recent discovery", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fault count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Caused by Change (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "MAC Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Schedule", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Supported by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Support group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Justification", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Change Request (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Incident state (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Month Tooltip", + "nullable": false, + "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problem (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Checked in", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Severity (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost center", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Subcategory (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Can Print", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolved (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time to Close an Incident", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),Custom SQL Query)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),Custom SQL Query)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_id)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_last_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),payment_date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_first_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "test publish datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "test publish datasource", + "description": "description for test publish datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Published SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Published Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)" + ], + "transformOperation": "HierarchyField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)" + ], + "transformOperation": "GroupField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)" + ], + "transformOperation": "HierarchyField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)" + ], + "transformOperation": "BinField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)" + ], + "transformOperation": "CalculatedFieldformula: SUM([Profit])/SUM([Sales])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD),Segment)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)" + ], + "transformOperation": "SetField", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Samples" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Superstore Datasource", + "description": "Description for Superstore dataset", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Top Customers by Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:SETFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returns", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit Ratio", + "nullable": false, + "description": "formula: SUM([Profit])/SUM([Sales])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Orders", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit (bin)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:BINFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID (Returns)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Person", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "People", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:GROUPFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region (People)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Published Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "urn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Customer Payment Query" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT\n\tcustomer.customer_id,\n\tfirst_name,\n\tlast_name,\n\tamount,\n\tpayment_date,\n\trental_id\nFROM\n\tcustomer\nINNER JOIN payment \n ON payment.customer_id = customer.customer_id\nwhere customer.customer_id = <[Parameters].[Parameter 1]>\nORDER BY payment_date", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/test publish datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT\n\tc.customer_id,\n\tc.first_name customer_first_name,\n\tc.last_name customer_last_name,\n\ts.first_name staff_first_name,\n\ts.last_name staff_last_name,\n\tamount,\n\tpayment_date\nFROM\n\tcustomer c\nINNER JOIN payment p \n ON p.customer_id = c.customer_id\nINNER JOIN staff s \n ON p.staff_id = s.staff_id\nORDER BY payment_date", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "test publish datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,demo-custom-323403.bigquery_demo.order_items,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,demo-custom-323403.bigquery_demo.sellers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD),seller_city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD),price)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "seller_city", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/SubProject1/AbcJoinWorkbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "select seller_city, price from demo-custom-323403.bigquery_demo.sellers sell LEFT JOIN (\nselect * from demo-custom-323403.bigquery_demo.order_items\n) items on items.seller_id=sell.seller_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "SubProject1" + }, + { + "id": "AbcJoinWorkbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Link", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Platform", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Link_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Mobile_Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User_Agent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Platform", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Mobile_Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User_Agent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "programName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "programId", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "createdAt", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "workspaceName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "updatedAt", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook/actor+ (dvdrental)" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "postal_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "phone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "city_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I2", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "district", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook/actor+ (dvdrental)" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "actor_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Person", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Returned", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Product ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Problems", + "/prod/tableau/default/Executive Dashboard/Requests", + "/prod/tableau/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "requested_for", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "requested_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "request_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "stage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "special_instructions", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "configuration_item", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_catalog", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "stage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "estimated_delivery", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "context", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billable", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_frequency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cat_item", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order_guide", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "backordered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "request", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sc_catalogs", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_picture_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "workflow", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_customer_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_standalone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_scope", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "template", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_proceed_checkout", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billable", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "meta", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ordered_item_link", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_ic_version", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "image", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_policy", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "roles", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "picture", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "list_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_order_now", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_ic_item_staging", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "entitlement_script", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "icon", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ignore_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "start_closed", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_package", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_replace_on_upgrade", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_FLOAT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "use_sc_layout", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "availability", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "custom_cart", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_picture", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_cart", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_hide_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_time", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_search", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_frequency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan_script", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_bundle", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_update_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_guide", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "preview", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "omit_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "u_u", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "source", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "exclude_manager", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_center", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "u_lucha", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "u_lu2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "roles", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "default_assignee", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "manager", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "include_members", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "related_incidents", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rfc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "problem_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_around", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "known_error", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "severity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent_incident", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "subcategory", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "caller_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "resolved_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "resolved_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "child_incidents", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "incident_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "notify", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "problem_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "caused_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reopen_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rfc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "first_discovered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "operational_status", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_discovered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_cc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "checked_in", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "attributes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "serial_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ip_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "support_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "asset", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "supported_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "invoice_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "managed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fault_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_in", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "justification", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "lease_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "monitor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_center", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "subcategory", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "can_print", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "start_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "discovery_source", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "schedule", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fqdn", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warranty_expiration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "owned_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "asset_tag", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "manufacturer", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "purchase_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "department", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "checked_out", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "unverified", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "skip_sync", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "po_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "gl_account", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "maintenance_schedule", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "install_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "dns_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mac_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "change_control", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "install_status", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Customer Payment Query", + "/prod/tableau/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Customer Payment Query", + "/prod/tableau/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "test publish datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:ATTRIBUTE", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "ATTRIBUTE" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:BINFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "BINFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:CALCULATEDFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "CALCULATEDFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:COLUMNFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "COLUMNFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:COUNT", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "COUNT" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:DATASOURCEFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "DATASOURCEFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:DIMENSION", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "DIMENSION" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:GROUPFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "GROUPFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:HIERARCHYFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "HIERARCHYFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:MEASURE", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "MEASURE" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:SETFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "SETFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:SUM", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "SUM" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:TagSheet3", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "TagSheet3" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:YEAR", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "YEAR" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json index e6782c1c9f5f3..12eb780447c18 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json @@ -12779,6 +12779,14 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "type": "TRANSFORMED" } ], "fineGrainedLineages": [ @@ -12796,7 +12804,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),last_name)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD),last_name)" ], "downstreamType": "FIELD", "downstreams": [ @@ -33581,6 +33589,132 @@ "runId": "tableau-test" } }, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "seller_city", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/SubProject1/AbcJoinWorkbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "select seller_city, price from demo-custom-323403.bigquery_demo.sellers sell LEFT JOIN (\nselect * from demo-custom-323403.bigquery_demo.order_items\n) items on items.seller_id=sell.seller_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "SubProject1" + }, + { + "id": "AbcJoinWorkbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -42756,6 +42890,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", diff --git a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json index ea15a2ac56b58..77ea484abfa0e 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json @@ -12547,6 +12547,14 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "type": "TRANSFORMED" } ], "fineGrainedLineages": [ @@ -12564,7 +12572,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),last_name)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD),last_name)" ], "downstreamType": "FIELD", "downstreams": [ @@ -33349,6 +33357,132 @@ "runId": "tableau-test" } }, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "seller_city", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/SubProject1/AbcJoinWorkbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "select seller_city, price from demo-custom-323403.bigquery_demo.sellers sell LEFT JOIN (\nselect * from demo-custom-323403.bigquery_demo.order_items\n) items on items.seller_id=sell.seller_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "SubProject1" + }, + { + "id": "AbcJoinWorkbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -42524,6 +42658,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", diff --git a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden_deleted_stateful.json b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden_deleted_stateful.json index 08d5127b18041..fb9cca0b6a544 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden_deleted_stateful.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden_deleted_stateful.json @@ -482,6 +482,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": true + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", diff --git a/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json index e6782c1c9f5f3..87cba090cee77 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json @@ -12779,6 +12779,14 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "type": "TRANSFORMED" } ], "fineGrainedLineages": [ @@ -12796,7 +12804,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),last_name)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD),last_name)" ], "downstreamType": "FIELD", "downstreams": [ @@ -33581,6 +33589,103 @@ "runId": "tableau-test" } }, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "seller_city", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "select seller_city, price from demo-custom-323403.bigquery_demo.sellers sell LEFT JOIN (\nselect * from demo-custom-323403.bigquery_demo.order_items\n) items on items.seller_id=sell.seller_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -42756,6 +42861,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", diff --git a/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json index ea15a2ac56b58..77ea484abfa0e 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json @@ -12547,6 +12547,14 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "type": "TRANSFORMED" } ], "fineGrainedLineages": [ @@ -12564,7 +12572,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),last_name)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD),last_name)" ], "downstreamType": "FIELD", "downstreams": [ @@ -33349,6 +33357,132 @@ "runId": "tableau-test" } }, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "seller_city", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/SubProject1/AbcJoinWorkbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "select seller_city, price from demo-custom-323403.bigquery_demo.sellers sell LEFT JOIN (\nselect * from demo-custom-323403.bigquery_demo.order_items\n) items on items.seller_id=sell.seller_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "SubProject1" + }, + { + "id": "AbcJoinWorkbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -42524,6 +42658,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", diff --git a/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json index e94ff0118e04c..7408e481a9411 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json @@ -12649,6 +12649,14 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "type": "TRANSFORMED" } ], "fineGrainedLineages": [ @@ -12666,7 +12674,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),last_name)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.10c6297d-0dbd-44f1-b1ba-458bea446513,PROD),last_name)" ], "downstreamType": "FIELD", "downstreams": [ @@ -33490,6 +33498,137 @@ "runId": "tableau-test" } }, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "seller_city", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/SubProject1/AbcJoinWorkbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "select seller_city, price from demo-custom-323403.bigquery_demo.sellers sell LEFT JOIN (\nselect * from demo-custom-323403.bigquery_demo.order_items\n) items on items.seller_id=sell.seller_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" + }, + { + "id": "SubProject1" + }, + { + "id": "AbcJoinWorkbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -42749,6 +42888,21 @@ "runId": "tableau-test" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,acryl_site1.22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", diff --git a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py index 8615545ffbd57..d04c8d905b439 100644 --- a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py +++ b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py @@ -2,7 +2,7 @@ import logging import pathlib import sys -from typing import cast +from typing import Any, Dict, cast from unittest import mock import pytest @@ -27,6 +27,7 @@ UpstreamLineage, ) from datahub.metadata.schema_classes import MetadataChangeProposalClass, UpstreamClass +from datahub.utilities.sqlglot_lineage import SqlParsingResult from tests.test_helpers import mce_helpers from tests.test_helpers.state_helpers import ( get_current_checkpoint_from_pipeline, @@ -289,6 +290,39 @@ def test_tableau_ingest(pytestconfig, tmp_path, mock_datahub_graph): ) +@freeze_time(FROZEN_TIME) +@pytest.mark.integration +def test_tableau_cll_ingest(pytestconfig, tmp_path, mock_datahub_graph): + enable_logging() + output_file_name: str = "tableau_mces_cll.json" + golden_file_name: str = "tableau_cll_mces_golden.json" + + new_pipeline_config: Dict[Any, Any] = { + **config_source_default, + "extract_lineage_from_unsupported_custom_sql_queries": True, + "extract_column_level_lineage": True, + } + + tableau_ingest_common( + pytestconfig=pytestconfig, + tmp_path=tmp_path, + side_effect_query_metadata_response=[ + read_response(pytestconfig, "workbooksConnection_all.json"), + read_response(pytestconfig, "sheetsConnection_all.json"), + read_response(pytestconfig, "dashboardsConnection_all.json"), + read_response(pytestconfig, "embeddedDatasourcesConnection_all.json"), + read_response(pytestconfig, "publishedDatasourcesConnection_all.json"), + read_response(pytestconfig, "customSQLTablesConnection_all.json"), + read_response(pytestconfig, "databaseTablesConnection_all.json"), + ], + golden_file_name=golden_file_name, + output_file_name=output_file_name, + mock_datahub_graph=mock_datahub_graph, + pipeline_name="test_tableau_cll_ingest", + pipeline_config=new_pipeline_config, + ) + + @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_project_pattern(pytestconfig, tmp_path, mock_datahub_graph): @@ -615,7 +649,7 @@ def test_tableau_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph) state1.get_urns_not_in(type="dataset", other_checkpoint_state=state2) ) - assert len(difference_dataset_urns) == 33 + assert len(difference_dataset_urns) == 34 deleted_dataset_urns = [ "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", @@ -650,6 +684,7 @@ def test_tableau_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph) "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", ] assert sorted(deleted_dataset_urns) == sorted(difference_dataset_urns) @@ -755,30 +790,48 @@ def test_tableau_unsupported_csql(mock_datahub_graph): config.lineage_overrides = TableauLineageOverrides( database_override_map={"production database": "prod"} ) - source = TableauSource(config=config, ctx=context) - lineage = source._create_lineage_from_unsupported_csql( - csql_urn="urn:li:dataset:(urn:li:dataPlatform:tableau,09988088-05ad-173c-a2f1-f33ba3a13d1a,PROD)", - csql={ - "query": "SELECT user_id, source, user_source FROM (SELECT *, ROW_NUMBER() OVER (partition BY user_id ORDER BY __partition_day DESC) AS rank_ FROM invent_dw.UserDetail ) source_user WHERE rank_ = 1", - "isUnsupportedCustomSql": "true", - "database": {"name": "production database", "connectionType": "bigquery"}, - }, - ) - mcp = cast(MetadataChangeProposalClass, next(iter(lineage)).metadata) + with mock.patch( + "datahub.ingestion.source.tableau.sqlglot_lineage" + ) as sqlglot_lineage: - assert mcp.aspect == UpstreamLineage( - upstreams=[ - UpstreamClass( - dataset="urn:li:dataset:(urn:li:dataPlatform:bigquery,prod.invent_dw.userdetail,PROD)", - type=DatasetLineageType.TRANSFORMED, - ) - ] - ) - assert ( - mcp.entityUrn - == "urn:li:dataset:(urn:li:dataPlatform:tableau,09988088-05ad-173c-a2f1-f33ba3a13d1a,PROD)" - ) + sqlglot_lineage.return_value = SqlParsingResult( # type:ignore + in_tables=[ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,my_bigquery_project.invent_dw.userdetail,PROD)" + ], + out_tables=[], + column_lineage=None, + ) + + source = TableauSource(config=config, ctx=context) + + lineage = source._create_lineage_from_unsupported_csql( + csql_urn="urn:li:dataset:(urn:li:dataPlatform:tableau,09988088-05ad-173c-a2f1-f33ba3a13d1a,PROD)", + csql={ + "query": "SELECT user_id, source, user_source FROM (SELECT *, ROW_NUMBER() OVER (partition BY user_id ORDER BY __partition_day DESC) AS rank_ FROM invent_dw.UserDetail ) source_user WHERE rank_ = 1", + "isUnsupportedCustomSql": "true", + "database": { + "name": "my-bigquery-project", + "connectionType": "bigquery", + }, + }, + ) + + mcp = cast(MetadataChangeProposalClass, next(iter(lineage)).metadata) + + assert mcp.aspect == UpstreamLineage( + upstreams=[ + UpstreamClass( + dataset="urn:li:dataset:(urn:li:dataPlatform:bigquery,my_bigquery_project.invent_dw.userdetail,PROD)", + type=DatasetLineageType.TRANSFORMED, + ) + ], + fineGrainedLineages=[], + ) + assert ( + mcp.entityUrn + == "urn:li:dataset:(urn:li:dataPlatform:tableau,09988088-05ad-173c-a2f1-f33ba3a13d1a,PROD)" + ) @freeze_time(FROZEN_TIME) From 2e2a6748acf8fc1cf1307e7bb4185a78b9931100 Mon Sep 17 00:00:00 2001 From: Benjamin Dornel <62495124+benjamin-awd@users.noreply.github.com> Date: Tue, 1 Aug 2023 22:05:40 +0800 Subject: [PATCH 216/222] fix(ingest/json-schema): convert non-string enums to strings (#8479) --- .../datahub/ingestion/extractor/json_schema_util.py | 4 +++- .../tests/unit/schema/test_json_schema_util.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py b/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py index cfa1b7609c0b1..8e313e92cbf84 100644 --- a/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py +++ b/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py @@ -254,11 +254,13 @@ def _field_from_primitive( isPartOfKey=field_path.is_key_schema, ) elif datahub_field_type in [EnumTypeClass]: + # Convert enums to string representation + schema_enums = list(map(json.dumps, schema["enum"])) yield SchemaField( fieldPath=field_path.expand_type("enum", schema).as_string(), type=type_override or SchemaFieldDataTypeClass(type=EnumTypeClass()), nativeDataType="Enum", - description=f"one of {','.join(schema['enum'])}", + description=f"One of: {', '.join(schema_enums)}", nullable=nullable, jsonProps=JsonSchemaTranslator._get_jsonprops_for_any_schema( schema, required=required diff --git a/metadata-ingestion/tests/unit/schema/test_json_schema_util.py b/metadata-ingestion/tests/unit/schema/test_json_schema_util.py index 731ee336621fd..2635363ed8d2e 100644 --- a/metadata-ingestion/tests/unit/schema/test_json_schema_util.py +++ b/metadata-ingestion/tests/unit/schema/test_json_schema_util.py @@ -712,6 +712,19 @@ def test_required_field(): assert json.loads(fields[0].jsonProps or "{}")["required"] is False +def test_non_str_enums(): + schema = { + "$id": "test", + "$schema": "http://json-schema.org/draft-06/schema#", + "properties": {"bar": {"description": "Mixed enum", "enum": ["baz", 1, None]}}, + } + + fields = list(JsonSchemaTranslator.get_fields_from_schema(schema)) + expected_field_paths: List[str] = ["[version=2.0].[type=object].[type=enum].bar"] + assert_field_paths_match(fields, expected_field_paths) + assert fields[0].description == 'One of: "baz", 1, null' + + def test_anyof_with_properties(): # We expect the event / timestamp fields to be included in both branches of the anyOf. From 9f791a3b6e6468eb32caa3118889275bd61a52f3 Mon Sep 17 00:00:00 2001 From: RyanHolstien Date: Tue, 1 Aug 2023 12:48:02 -0500 Subject: [PATCH 217/222] feat(browseV2): add browseV2 logic to system update (#8506) Co-authored-by: david-leifker <114954101+david-leifker@users.noreply.github.com> --- .../config/BackfillBrowsePathsV2Config.java | 17 ++ .../upgrade/config/SystemUpdateConfig.java | 6 +- .../datahub/upgrade/system/SystemUpdate.java | 6 +- .../entity/steps/BackfillBrowsePathsV2.java | 28 +++ .../steps/BackfillBrowsePathsV2Step.java | 167 ++++++++++++++++++ .../env/docker-without-neo4j.env | 1 + docker/datahub-upgrade/env/docker.env | 1 + .../docker-compose-m1.quickstart.yml | 1 + ...er-compose-without-neo4j-m1.quickstart.yml | 1 + ...ocker-compose-without-neo4j.quickstart.yml | 1 + .../quickstart/docker-compose.quickstart.yml | 1 + .../src/main/resources/application.yml | 2 +- 12 files changed, 227 insertions(+), 5 deletions(-) create mode 100644 datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BackfillBrowsePathsV2Config.java create mode 100644 datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2.java create mode 100644 datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2Step.java diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BackfillBrowsePathsV2Config.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BackfillBrowsePathsV2Config.java new file mode 100644 index 0000000000000..16e5e4247267f --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BackfillBrowsePathsV2Config.java @@ -0,0 +1,17 @@ +package com.linkedin.datahub.upgrade.config; + +import com.linkedin.datahub.upgrade.system.entity.steps.BackfillBrowsePathsV2; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.search.SearchService; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + + +@Configuration +public class BackfillBrowsePathsV2Config { + + @Bean + public BackfillBrowsePathsV2 backfillBrowsePathsV2(EntityService entityService, SearchService searchService) { + return new BackfillBrowsePathsV2(entityService, searchService); + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java index 826bd0180d037..9848fc7a0008f 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java @@ -3,6 +3,7 @@ import com.linkedin.datahub.upgrade.system.SystemUpdate; import com.linkedin.datahub.upgrade.system.elasticsearch.BuildIndices; import com.linkedin.datahub.upgrade.system.elasticsearch.CleanIndices; +import com.linkedin.datahub.upgrade.system.entity.steps.BackfillBrowsePathsV2; import com.linkedin.gms.factory.common.TopicConventionFactory; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.gms.factory.kafka.DataHubKafkaProducerFactory; @@ -30,10 +31,11 @@ public class SystemUpdateConfig { @Bean(name = "systemUpdate") public SystemUpdate systemUpdate(final BuildIndices buildIndices, final CleanIndices cleanIndices, @Qualifier("duheKafkaEventProducer") final KafkaEventProducer kafkaEventProducer, - final GitVersion gitVersion, @Qualifier("revision") String revision) { + final GitVersion gitVersion, @Qualifier("revision") String revision, + final BackfillBrowsePathsV2 backfillBrowsePathsV2) { String version = String.format("%s-%s", gitVersion.getVersion(), revision); - return new SystemUpdate(buildIndices, cleanIndices, kafkaEventProducer, version); + return new SystemUpdate(buildIndices, cleanIndices, kafkaEventProducer, version, backfillBrowsePathsV2); } @Value("#{systemEnvironment['DATAHUB_REVISION'] ?: '0'}") diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/SystemUpdate.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/SystemUpdate.java index d7e490501a9fb..4a8211f2cd4ac 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/SystemUpdate.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/SystemUpdate.java @@ -6,6 +6,7 @@ import com.linkedin.datahub.upgrade.system.elasticsearch.BuildIndices; import com.linkedin.datahub.upgrade.system.elasticsearch.CleanIndices; import com.linkedin.datahub.upgrade.system.elasticsearch.steps.DataHubStartupStep; +import com.linkedin.datahub.upgrade.system.entity.steps.BackfillBrowsePathsV2; import com.linkedin.metadata.dao.producer.KafkaEventProducer; import lombok.extern.slf4j.Slf4j; @@ -21,11 +22,12 @@ public class SystemUpdate implements Upgrade { private final List _steps; public SystemUpdate(final BuildIndices buildIndicesJob, final CleanIndices cleanIndicesJob, - final KafkaEventProducer kafkaEventProducer, final String version) { + final KafkaEventProducer kafkaEventProducer, final String version, + final BackfillBrowsePathsV2 backfillBrowsePathsV2) { _preStartupUpgrades = List.of(buildIndicesJob); _steps = List.of(new DataHubStartupStep(kafkaEventProducer, version)); - _postStartupUpgrades = List.of(cleanIndicesJob); + _postStartupUpgrades = List.of(cleanIndicesJob, backfillBrowsePathsV2); } @Override diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2.java new file mode 100644 index 0000000000000..e213c0b2fd4de --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2.java @@ -0,0 +1,28 @@ +package com.linkedin.datahub.upgrade.system.entity.steps; + +import com.google.common.collect.ImmutableList; +import com.linkedin.datahub.upgrade.Upgrade; +import com.linkedin.datahub.upgrade.UpgradeStep; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.search.SearchService; +import java.util.List; + + +public class BackfillBrowsePathsV2 implements Upgrade { + + private final List _steps; + + public BackfillBrowsePathsV2(EntityService entityService, SearchService searchService) { + _steps = ImmutableList.of(new BackfillBrowsePathsV2Step(entityService, searchService)); + } + + @Override + public String id() { + return "BackfillBrowsePathsV2"; + } + + @Override + public List steps() { + return _steps; + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2Step.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2Step.java new file mode 100644 index 0000000000000..7547186ccfb23 --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2Step.java @@ -0,0 +1,167 @@ +package com.linkedin.datahub.upgrade.system.entity.steps; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.BrowsePathsV2; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.upgrade.UpgradeContext; +import com.linkedin.datahub.upgrade.UpgradeStep; +import com.linkedin.datahub.upgrade.UpgradeStepResult; +import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.search.ScrollResult; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.search.SearchService; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.mxe.SystemMetadata; +import java.util.function.Function; +import lombok.extern.slf4j.Slf4j; + +import java.util.Set; + +import static com.linkedin.metadata.Constants.*; + + +@Slf4j +public class BackfillBrowsePathsV2Step implements UpgradeStep { + + public static final String BACKFILL_BROWSE_PATHS_V2 = "BACKFILL_BROWSE_PATHS_V2"; + + private static final Set ENTITY_TYPES_TO_MIGRATE = ImmutableSet.of( + Constants.DATASET_ENTITY_NAME, + Constants.DASHBOARD_ENTITY_NAME, + Constants.CHART_ENTITY_NAME, + Constants.DATA_JOB_ENTITY_NAME, + Constants.DATA_FLOW_ENTITY_NAME, + Constants.ML_MODEL_ENTITY_NAME, + Constants.ML_MODEL_GROUP_ENTITY_NAME, + Constants.ML_FEATURE_TABLE_ENTITY_NAME, + Constants.ML_FEATURE_ENTITY_NAME + ); + private static final Integer BATCH_SIZE = 5000; + + private final EntityService _entityService; + private final SearchService _searchService; + + public BackfillBrowsePathsV2Step(EntityService entityService, SearchService searchService) { + _searchService = searchService; + _entityService = entityService; + } + + @Override + public Function executable() { + return (context) -> { + final AuditStamp auditStamp = + new AuditStamp().setActor(UrnUtils.getUrn(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis()); + + String scrollId = null; + for (String entityType : ENTITY_TYPES_TO_MIGRATE) { + int migratedCount = 0; + do { + log.info(String.format("Upgrading batch %s-%s of browse paths for entity type %s", migratedCount, + migratedCount + BATCH_SIZE, entityType)); + scrollId = backfillBrowsePathsV2(entityType, auditStamp, scrollId); + migratedCount += BATCH_SIZE; + } while (scrollId != null); + } + return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + }; + } + + private String backfillBrowsePathsV2(String entityType, AuditStamp auditStamp, String scrollId) { + + // Condition: has `browsePaths` AND does NOT have `browsePathV2` + Criterion missingBrowsePathV2 = new Criterion(); + missingBrowsePathV2.setCondition(Condition.IS_NULL); + missingBrowsePathV2.setField("browsePathV2"); + // Excludes entities without browsePaths + Criterion hasBrowsePathV1 = new Criterion(); + hasBrowsePathV1.setCondition(Condition.EXISTS); + hasBrowsePathV1.setField("browsePaths"); + + CriterionArray criterionArray = new CriterionArray(); + criterionArray.add(missingBrowsePathV2); + criterionArray.add(hasBrowsePathV1); + + ConjunctiveCriterion conjunctiveCriterion = new ConjunctiveCriterion(); + conjunctiveCriterion.setAnd(criterionArray); + + ConjunctiveCriterionArray conjunctiveCriterionArray = new ConjunctiveCriterionArray(); + conjunctiveCriterionArray.add(conjunctiveCriterion); + + Filter filter = new Filter(); + filter.setOr(conjunctiveCriterionArray); + + final ScrollResult scrollResult = _searchService.scrollAcrossEntities( + ImmutableList.of(entityType), + "*", + filter, + null, + scrollId, + "5m", + BATCH_SIZE, + null + ); + if (scrollResult.getNumEntities() == 0 || scrollResult.getEntities().size() == 0) { + return null; + } + + for (SearchEntity searchEntity : scrollResult.getEntities()) { + try { + ingestBrowsePathsV2(searchEntity.getEntity(), auditStamp); + } catch (Exception e) { + // don't stop the whole step because of one bad urn or one bad ingestion + log.error(String.format("Error ingesting default browsePathsV2 aspect for urn %s", searchEntity.getEntity()), e); + } + } + + return scrollResult.getScrollId(); + } + + private void ingestBrowsePathsV2(Urn urn, AuditStamp auditStamp) throws Exception { + BrowsePathsV2 browsePathsV2 = _entityService.buildDefaultBrowsePathV2(urn, true); + log.debug(String.format("Adding browse path v2 for urn %s with value %s", urn, browsePathsV2)); + MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityUrn(urn); + proposal.setEntityType(urn.getEntityType()); + proposal.setAspectName(Constants.BROWSE_PATHS_V2_ASPECT_NAME); + proposal.setChangeType(ChangeType.UPSERT); + proposal.setSystemMetadata(new SystemMetadata().setRunId(DEFAULT_RUN_ID).setLastObserved(System.currentTimeMillis())); + proposal.setAspect(GenericRecordUtils.serializeAspect(browsePathsV2)); + _entityService.ingestProposal( + proposal, + auditStamp, + false + ); + } + + @Override + public String id() { + return "BackfillBrowsePathsV2Step"; + } + + /** + * Returns whether the upgrade should proceed if the step fails after exceeding the maximum retries. + */ + @Override + public boolean isOptional() { + return true; + } + + @Override + public boolean skip(UpgradeContext context) { + return !Boolean.parseBoolean(System.getenv(BACKFILL_BROWSE_PATHS_V2)); + } +} + diff --git a/docker/datahub-upgrade/env/docker-without-neo4j.env b/docker/datahub-upgrade/env/docker-without-neo4j.env index c253425897006..c399f71b7b15c 100644 --- a/docker/datahub-upgrade/env/docker-without-neo4j.env +++ b/docker/datahub-upgrade/env/docker-without-neo4j.env @@ -20,6 +20,7 @@ DATAHUB_GMS_HOST=datahub-gms DATAHUB_GMS_PORT=8080 ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml +BACKFILL_BROWSE_PATHS_V2=true # Uncomment and set these to support SSL connection to Elasticsearch # ELASTICSEARCH_USE_SSL= diff --git a/docker/datahub-upgrade/env/docker.env b/docker/datahub-upgrade/env/docker.env index f8cf050e06154..491470406153b 100644 --- a/docker/datahub-upgrade/env/docker.env +++ b/docker/datahub-upgrade/env/docker.env @@ -24,6 +24,7 @@ DATAHUB_GMS_HOST=datahub-gms DATAHUB_GMS_PORT=8080 ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml +BACKFILL_BROWSE_PATHS_V2=true # Uncomment and set these to support SSL connection to Elasticsearch # ELASTICSEARCH_USE_SSL= diff --git a/docker/quickstart/docker-compose-m1.quickstart.yml b/docker/quickstart/docker-compose-m1.quickstart.yml index 89569510bb001..0e6b06ec5de0b 100644 --- a/docker/quickstart/docker-compose-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-m1.quickstart.yml @@ -146,6 +146,7 @@ services: - DATAHUB_GMS_HOST=datahub-gms - DATAHUB_GMS_PORT=8080 - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - BACKFILL_BROWSE_PATHS_V2=true hostname: datahub-upgrade image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head} labels: diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index 102500c8f1878..ba2df82ed9030 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -139,6 +139,7 @@ services: - DATAHUB_GMS_HOST=datahub-gms - DATAHUB_GMS_PORT=8080 - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - BACKFILL_BROWSE_PATHS_V2=true hostname: datahub-upgrade image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head} labels: diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index 5fa2838255d9a..426e36d80f48c 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -139,6 +139,7 @@ services: - DATAHUB_GMS_HOST=datahub-gms - DATAHUB_GMS_PORT=8080 - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - BACKFILL_BROWSE_PATHS_V2=true hostname: datahub-upgrade image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head} labels: diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index adee2da9276aa..837eaeb15007d 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -146,6 +146,7 @@ services: - DATAHUB_GMS_HOST=datahub-gms - DATAHUB_GMS_PORT=8080 - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - BACKFILL_BROWSE_PATHS_V2=true hostname: datahub-upgrade image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head} labels: diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml index 5290dd4adceac..dfc7218eb772d 100644 --- a/metadata-service/configuration/src/main/resources/application.yml +++ b/metadata-service/configuration/src/main/resources/application.yml @@ -270,7 +270,7 @@ bootstrap: upgradeDefaultBrowsePaths: enabled: ${UPGRADE_DEFAULT_BROWSE_PATHS_ENABLED:false} # enable to run the upgrade to migrate legacy default browse paths to new ones backfillBrowsePathsV2: - enabled: ${BACKFILL_BROWSE_PATHS_V2:false} # Enables running the backfill of browsePathsV2 upgrade step. There are concerns about the load of this step so hiding it behind a flag. + enabled: ${BACKFILL_BROWSE_PATHS_V2:false} # Enables running the backfill of browsePathsV2 upgrade step. There are concerns about the load of this step so hiding it behind a flag. Deprecating in favor of running through SystemUpdate systemUpdate: initialBackOffMs: ${BOOTSTRAP_SYSTEM_UPDATE_INITIAL_BACK_OFF_MILLIS:5000} From a4a818200123aef11139145b4708cc71e635d336 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Tue, 1 Aug 2023 21:35:42 +0100 Subject: [PATCH 218/222] feat(cli): Adds ability to upload recipes to DataHub's UI (#8317) Co-authored-by: Indy Prentice --- .../source/ListIngestionSourcesResolver.java | 6 +- .../src/main/resources/ingestion.graphql | 5 + .../ListIngestionSourceResolverTest.java | 6 +- docs/cli.md | 26 ++- docs/ui-ingestion.md | 42 +++++ metadata-ingestion/README.md | 13 ++ metadata-ingestion/setup.py | 1 + .../src/datahub/cli/ingest_cli.py | 153 ++++++++++++++++++ .../datahub/configuration/config_loader.py | 6 +- 9 files changed, 248 insertions(+), 10 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourcesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourcesResolver.java index fa56f15bcf8d4..d019473606e58 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourcesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourcesResolver.java @@ -4,6 +4,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.ListIngestionSourcesInput; import com.linkedin.datahub.graphql.generated.ListIngestionSourcesResult; import com.linkedin.datahub.graphql.resolvers.ingest.IngestionAuthUtils; @@ -20,6 +21,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; @@ -51,6 +53,7 @@ public CompletableFuture get(final DataFetchingEnvir final Integer start = input.getStart() == null ? DEFAULT_START : input.getStart(); final Integer count = input.getCount() == null ? DEFAULT_COUNT : input.getCount(); final String query = input.getQuery() == null ? DEFAULT_QUERY : input.getQuery(); + final List filters = input.getFilters() == null ? Collections.emptyList() : input.getFilters(); return CompletableFuture.supplyAsync(() -> { try { @@ -58,7 +61,8 @@ public CompletableFuture get(final DataFetchingEnvir final SearchResult gmsResult = _entityClient.search( Constants.INGESTION_SOURCE_ENTITY_NAME, query, - Collections.emptyMap(), + buildFilter(filters, Collections.emptyList()), + null, start, count, context.getAuthentication(), diff --git a/datahub-graphql-core/src/main/resources/ingestion.graphql b/datahub-graphql-core/src/main/resources/ingestion.graphql index 256b94ccdc244..69c8aff124583 100644 --- a/datahub-graphql-core/src/main/resources/ingestion.graphql +++ b/datahub-graphql-core/src/main/resources/ingestion.graphql @@ -428,6 +428,11 @@ input ListIngestionSourcesInput { An optional search query """ query: String + + """ + Optional Facet filters to apply to the result set + """ + filters: [FacetFilterInput!] } """ diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourceResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourceResolverTest.java index 3fa19cca0623c..8e2453ce06a39 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourceResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourceResolverTest.java @@ -19,7 +19,6 @@ import com.linkedin.metadata.search.SearchResult; import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetchingEnvironment; -import java.util.Collections; import java.util.HashSet; import org.mockito.Mockito; import org.testng.annotations.Test; @@ -30,7 +29,7 @@ public class ListIngestionSourceResolverTest { - private static final ListIngestionSourcesInput TEST_INPUT = new ListIngestionSourcesInput(0, 20, null); + private static final ListIngestionSourcesInput TEST_INPUT = new ListIngestionSourcesInput(0, 20, null, null); @Test public void testGetSuccess() throws Exception { @@ -44,7 +43,8 @@ public void testGetSuccess() throws Exception { Mockito.when(mockClient.search( Mockito.eq(Constants.INGESTION_SOURCE_ENTITY_NAME), Mockito.eq(""), - Mockito.eq(Collections.emptyMap()), + Mockito.any(), + Mockito.any(), Mockito.eq(0), Mockito.eq(20), Mockito.any(Authentication.class), diff --git a/docs/cli.md b/docs/cli.md index 64b7e2d76bba2..eb8bb406b0107 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -92,13 +92,29 @@ Source specific crawlers are provided by plugins and might sometimes need additi Usage: datahub [datahub-options] ingest [command-options] Command Options: - -c / --config Config file in .toml or .yaml format - -n / --dry-run Perform a dry run of the ingestion, essentially skipping writing to sink - --preview Perform limited ingestion from the source to the sink to get a quick preview - --preview-workunits The number of workunits to produce for preview - --strict-warnings If enabled, ingestion runs with warnings will yield a non-zero error code + -c / --config Config file in .toml or .yaml format + -n / --dry-run Perform a dry run of the ingestion, essentially skipping writing to sink + --preview Perform limited ingestion from the source to the sink to get a quick preview + --preview-workunits The number of workunits to produce for preview + --strict-warnings If enabled, ingestion runs with warnings will yield a non-zero error code + --test-source-connection When set, ingestion will only test the source connection details from the recipe ``` +#### ingest deploy + +The `ingest deploy` command instructs the cli to upload an ingestion recipe to DataHub to be run by DataHub's [UI Ingestion](./ui-ingestion.md). +This command can also be used to schedule the ingestion while uploading or even to update existing sources. + +To schedule a recipe called "test", to run at 5am everyday, London time with the recipe configured in a local `recipe.yaml` file: +````shell +datahub ingest deploy --name "test" --schedule "5 * * * *" --time-zone "Europe/London" -c recipe.yaml +```` + +To update an existing recipe please use the `--urn` parameter to specify the id of the recipe to update. + +**Note:** Updating a recipe will result in a replacement of the existing options with what was specified in the cli command. +I.e: Not specifying a schedule in the cli update command will remove the schedule from the recipe to be updated. + ### init The init command is used to tell `datahub` about where your DataHub instance is located. The CLI will point to localhost DataHub by default. diff --git a/docs/ui-ingestion.md b/docs/ui-ingestion.md index 235f1521c070a..4435f66e514f3 100644 --- a/docs/ui-ingestion.md +++ b/docs/ui-ingestion.md @@ -28,6 +28,9 @@ your first **Ingestion Source**. ### Creating an Ingestion Source + + + Before ingesting any metadata, you need to create a new Ingestion Source. Start by clicking **+ Create new source**. ![](./imgs/create-new-ingestion-source-button.png) @@ -151,6 +154,45 @@ _Pinning the CLI version to version `0.8.23.2`_ Once you're happy with your changes, simply click 'Done' to save. + + + +You can upload and even update recipes using the cli as mentioned in the [cli documentation for uploading ingestion recipes](./cli.md#ingest-deploy). +An example execution would look something like: + +```bash +datahub ingest deploy --name "My Test Ingestion Source" --schedule "5 * * * *" --time-zone "UTC" -c recipe.yaml +``` + +This would create a new recipe with the name `My Test Ingestion Source`. Note that to update an existing recipe, it's `urn` id must be passed as a parameter. +DataHub supports having multiple recipes with the same name so to distinguish them we use the urn for unique identification. + + + + +Create ingestion sources using [DataHub's GraphQL API](./api/graphql/overview.md) using the **createIngestionSource** mutation endpoint. +```graphql +mutation { + createIngestionSource(input: { + name: "My Test Ingestion Source", + type: "mysql", + description: "My ingestion source description", + schedule: {interval: "*/5 * * * *", timezone: "UTC"}, + config: { + recipe: "{\"source\":{\"type\":\"mysql\",\"config\":{\"include_tables\":true,\"database\":null,\"password\":\"${MYSQL_PASSWORD}\",\"profiling\":{\"enabled\":false},\"host_port\":null,\"include_views\":true,\"username\":\"${MYSQL_USERNAME}\"}},\"pipeline_name\":\"urn:li:dataHubIngestionSource:f38bd060-4ea8-459c-8f24-a773286a2927\"}", + version: "0.8.18", + executorId: "mytestexecutor", + } + }) +} +``` + +To update sources, please use the `updateIngestionSource` endpoint. It is almost identical to the create endpoint, only requiring the urn of the source to be updated in addition to the same input as the create endpoint. + +**Note**: Recipe must be double quotes escaped + + + ### Running an Ingestion Source diff --git a/metadata-ingestion/README.md b/metadata-ingestion/README.md index 47bc3b542163a..cacb5011d0766 100644 --- a/metadata-ingestion/README.md +++ b/metadata-ingestion/README.md @@ -161,6 +161,19 @@ reporting: report_recipe: false ``` +#### Deploying and scheduling ingestion to the UI + +The `deploy` subcommand of the `ingest` command tree allows users to upload their recipes and schedule them in the server. + +```shell +datahub ingest deploy -n -c recipe.yaml +``` + +By default, no schedule is done unless explicitly configured with the `--schedule` parameter. Timezones are inferred from the system time, can be overriden with `--time-zone` flag. +```shell +datahub ingest deploy -n test --schedule "0 * * * *" --time-zone "Europe/London" -c recipe.yaml +``` + ## Transformations If you'd like to modify data before it reaches the ingestion sinks – for instance, adding additional owners or tags – you can use a transformer to write your own module and integrate it with DataHub. Transformers require extending the recipe with a new section to describe the transformers that you want to run. diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 9940e8cdd0a97..4f156591eb756 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -415,6 +415,7 @@ def get_long_description(): "types-termcolor>=1.0.0", "types-Deprecated", "types-protobuf>=4.21.0.1", + "types-tzlocal", } diff --git a/metadata-ingestion/src/datahub/cli/ingest_cli.py b/metadata-ingestion/src/datahub/cli/ingest_cli.py index 72c15e92257aa..42c0ea1601c74 100644 --- a/metadata-ingestion/src/datahub/cli/ingest_cli.py +++ b/metadata-ingestion/src/datahub/cli/ingest_cli.py @@ -4,11 +4,13 @@ import logging import os import sys +import textwrap from datetime import datetime from typing import Optional import click import click_spinner +import tzlocal from click_default_group import DefaultGroup from tabulate import tabulate @@ -21,6 +23,7 @@ post_rollback_endpoint, ) from datahub.configuration.config_loader import load_config_file +from datahub.ingestion.graph.client import get_default_graph from datahub.ingestion.run.connection import ConnectionManager from datahub.ingestion.run.pipeline import Pipeline from datahub.telemetry import telemetry @@ -198,6 +201,156 @@ async def run_ingestion_and_check_upgrade() -> int: # don't raise SystemExit if there's no error +@ingest.command() +@upgrade.check_upgrade +@telemetry.with_telemetry() +@click.option( + "-n", + "--name", + type=str, + help="Recipe Name", + required=True, +) +@click.option( + "-c", + "--config", + type=click.Path(dir_okay=False), + help="Config file in .toml or .yaml format.", + required=True, +) +@click.option( + "--urn", + type=str, + help="Urn of recipe to update", + required=False, +) +@click.option( + "--executor-id", + type=str, + default="default", + help="Executor id to route execution requests to. Do not use this unless you have configured a custom executor.", + required=False, +) +@click.option( + "--cli-version", + type=str, + help="Provide a custom CLI version to use for ingestion. By default will use server default.", + required=False, + default=None, +) +@click.option( + "--schedule", + type=str, + help="Cron definition for schedule. If none is provided, ingestion recipe will not be scheduled", + required=False, + default=None, +) +@click.option( + "--time-zone", + type=str, + help=f"Timezone for the schedule. By default uses the timezone of the current system: {tzlocal.get_localzone_name()}.", + required=False, + default=tzlocal.get_localzone_name(), +) +def deploy( + name: str, + config: str, + urn: str, + executor_id: str, + cli_version: str, + schedule: str, + time_zone: str, +) -> None: + """ + Deploy an ingestion recipe to your DataHub instance. + + The urn of the ingestion source will be based on the name parameter in the format: + urn:li:dataHubIngestionSource: + """ + + datahub_graph = get_default_graph() + + pipeline_config = load_config_file( + config, + allow_stdin=True, + resolve_env_vars=False, + ) + + graphql_query: str + + variables: dict = { + "urn": urn, + "name": name, + "type": pipeline_config["source"]["type"], + "schedule": {"interval": schedule, "timezone": time_zone}, + "recipe": json.dumps(pipeline_config), + "executorId": executor_id, + "version": cli_version, + } + + if urn: + if not datahub_graph.exists(urn): + logger.error(f"Could not find recipe for provided urn: {urn}") + exit() + logger.info("Found recipe URN, will update recipe.") + + graphql_query = textwrap.dedent( + """ + mutation updateIngestionSource( + $urn: String!, + $name: String!, + $type: String!, + $schedule: UpdateIngestionSourceScheduleInput, + $recipe: String!, + $executorId: String! + $version: String) { + + updateIngestionSource(urn: $urn, input: { + name: $name, + type: $type, + schedule: $schedule, + config: { + recipe: $recipe, + executorId: $executorId, + version: $version, + } + }) + } + """ + ) + else: + logger.info("No URN specified recipe urn, will create a new recipe.") + graphql_query = textwrap.dedent( + """ + mutation createIngestionSource( + $name: String!, + $type: String!, + $schedule: UpdateIngestionSourceScheduleInput, + $recipe: String!, + $executorId: String!, + $version: String) { + + createIngestionSource(input: { + type: $type, + schedule: $schedule, + config: { + recipe: $recipe, + executorId: $executorId, + version: $version, + } + }) + } + """ + ) + + response = datahub_graph.execute_graphql(graphql_query, variables=variables) + + click.echo( + f"✅ Successfully wrote data ingestion source metadata for recipe {name}:" + ) + click.echo(response) + + def _test_source_connection(report_to: Optional[str], pipeline_config: dict) -> None: connection_report = None try: diff --git a/metadata-ingestion/src/datahub/configuration/config_loader.py b/metadata-ingestion/src/datahub/configuration/config_loader.py index 8c2c635bb1df5..78bee21d1bda4 100644 --- a/metadata-ingestion/src/datahub/configuration/config_loader.py +++ b/metadata-ingestion/src/datahub/configuration/config_loader.py @@ -72,6 +72,7 @@ def load_config_file( squirrel_original_config: bool = False, squirrel_field: str = "__orig_config", allow_stdin: bool = False, + resolve_env_vars: bool = True, ) -> dict: config_mech: ConfigurationMechanism if allow_stdin and config_file == "-": @@ -104,7 +105,10 @@ def load_config_file( config_fp = io.StringIO(raw_config_file) raw_config = config_mech.load_config(config_fp) - config = resolve_env_variables(raw_config) + if resolve_env_vars: + config = resolve_env_variables(raw_config) + else: + config = raw_config if squirrel_original_config: config[squirrel_field] = raw_config return config From 843f82b943b48c2e340cdb643b44fd97443a062c Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Tue, 1 Aug 2023 14:05:50 -0700 Subject: [PATCH 219/222] feat(presto-on-hive): allow v1 fieldpaths in the presto-on-hive source (#8474) --- .../datahub/ingestion/source/csv_enricher.py | 2 +- .../ingestion/source/sql/presto_on_hive.py | 7 + .../ingestion/source/sql/sql_common.py | 10 + .../datahub/ingestion/source/sql/sql_utils.py | 31 + .../src/datahub/utilities/sqlglot_lineage.py | 4 +- .../src/datahub/utilities/urns/dataset_urn.py | 2 +- .../presto_on_hive_mces_golden_5.json | 1693 +++++++++++++++++ .../presto-on-hive/test_presto_on_hive.py | 14 +- 8 files changed, 1754 insertions(+), 9 deletions(-) create mode 100644 metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_5.json diff --git a/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py b/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py index e41c02b462662..f057862a343b1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py +++ b/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py @@ -438,7 +438,7 @@ def process_sub_resource_row( field_match = False for field_info in current_editable_schema_metadata.editableSchemaFieldInfo: if ( - DatasetUrn._get_simple_field_path_from_v2_field_path( + DatasetUrn.get_simple_field_path_from_v2_field_path( field_info.fieldPath ) == field_path diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py b/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py index 55d4ebc5ffa44..a54cb9d50e2ae 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py @@ -134,6 +134,11 @@ class PrestoOnHiveConfig(BasicSQLAlchemyConfig): description="By default, the connector overwrites properties every time. Set this to True to enable merging of properties with what exists on the server.", ) + simplify_nested_field_paths: bool = Field( + default=False, + description="Simplify v2 field paths to v1 by default. If the schema has Union or Array types, still falls back to v2", + ) + def get_sql_alchemy_url( self, uri_opts: Optional[Dict[str, Any]] = None, database: Optional[str] = None ) -> str: @@ -527,6 +532,7 @@ def loop_tables( None, None, schema_fields, + self.config.simplify_nested_field_paths, ) dataset_snapshot.aspects.append(schema_metadata) @@ -756,6 +762,7 @@ def loop_views( self.platform, dataset.columns, canonical_schema=schema_fields, + simplify_nested_field_paths=self.config.simplify_nested_field_paths, ) dataset_snapshot.aspects.append(schema_metadata) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py index 42ea7aed9b620..4869df75d4fb8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py @@ -41,11 +41,13 @@ from datahub.ingestion.source.sql.sql_config import SQLAlchemyConfig from datahub.ingestion.source.sql.sql_utils import ( add_table_to_schema_container, + downgrade_schema_from_v2, gen_database_container, gen_database_key, gen_schema_container, gen_schema_key, get_domain_wu, + schema_requires_v2, ) from datahub.ingestion.source.state.stale_entity_removal_handler import ( StaleEntityRemovalHandler, @@ -287,7 +289,15 @@ def get_schema_metadata( pk_constraints: Optional[dict] = None, foreign_keys: Optional[List[ForeignKeyConstraint]] = None, canonical_schema: Optional[List[SchemaField]] = None, + simplify_nested_field_paths: bool = False, ) -> SchemaMetadata: + if ( + simplify_nested_field_paths + and canonical_schema is not None + and not schema_requires_v2(canonical_schema) + ): + canonical_schema = downgrade_schema_from_v2(canonical_schema) + schema_metadata = SchemaMetadata( schemaName=dataset_name, platform=make_data_platform_urn(platform), diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_utils.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_utils.py index a5f5034d175c6..c5baf148b0e5e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_utils.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_utils.py @@ -17,9 +17,16 @@ ) from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.metadata.com.linkedin.pegasus2avro.dataset import UpstreamLineage +from datahub.metadata.com.linkedin.pegasus2avro.schema import SchemaField from datahub.metadata.schema_classes import DataPlatformInstanceClass from datahub.specific.dataset import DatasetPatchBuilder from datahub.utilities.registries.domain_registry import DomainRegistry +from datahub.utilities.urns.dataset_urn import DatasetUrn + +ARRAY_TOKEN = "[type=array]" +UNION_TOKEN = "[type=union]" +KEY_SCHEMA_PREFIX = "[key=True]." +VERSION_PREFIX = "[version=2.0]." def gen_schema_key( @@ -223,3 +230,27 @@ def gen_lineage( for wu in lineage_workunits: yield wu + + +# downgrade a schema field +def downgrade_schema_field_from_v2(field: SchemaField) -> SchemaField: + field.fieldPath = DatasetUrn.get_simple_field_path_from_v2_field_path( + field.fieldPath + ) + return field + + +# downgrade a list of schema fields +def downgrade_schema_from_v2( + canonical_schema: List[SchemaField], +) -> List[SchemaField]: + return [downgrade_schema_field_from_v2(field) for field in canonical_schema] + + +# v2 is only required in case UNION or ARRAY types are present- all other types can be represented in v1 paths +def schema_requires_v2(canonical_schema: List[SchemaField]) -> bool: + for field in canonical_schema: + field_name = field.fieldPath + if ARRAY_TOKEN in field_name or UNION_TOKEN in field_name: + return True + return False diff --git a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py index 57f93f27e9147..e5a9954802019 100644 --- a/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py +++ b/metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py @@ -328,7 +328,7 @@ def _convert_schema_aspect_to_info( cls, schema_metadata: SchemaMetadataClass ) -> SchemaInfo: return { - DatasetUrn._get_simple_field_path_from_v2_field_path(col.fieldPath): ( + DatasetUrn.get_simple_field_path_from_v2_field_path(col.fieldPath): ( # The actual types are more of a "nice to have". col.nativeDataType or "str" @@ -336,7 +336,7 @@ def _convert_schema_aspect_to_info( for col in schema_metadata.fields # TODO: We can't generate lineage to columns nested within structs yet. if "." - not in DatasetUrn._get_simple_field_path_from_v2_field_path(col.fieldPath) + not in DatasetUrn.get_simple_field_path_from_v2_field_path(col.fieldPath) } # TODO add a method to load all from graphql diff --git a/metadata-ingestion/src/datahub/utilities/urns/dataset_urn.py b/metadata-ingestion/src/datahub/utilities/urns/dataset_urn.py index 283d4ac926cb8..3ed33c068496e 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/dataset_urn.py +++ b/metadata-ingestion/src/datahub/utilities/urns/dataset_urn.py @@ -97,7 +97,7 @@ def _validate_entity_id(entity_id: List[str]) -> None: """A helper function to extract simple . path notation from the v2 field path""" @staticmethod - def _get_simple_field_path_from_v2_field_path(field_path: str) -> str: + def get_simple_field_path_from_v2_field_path(field_path: str) -> str: if field_path.startswith("[version=2.0]"): # this is a v2 field path tokens = [ diff --git a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_5.json b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_5.json new file mode 100644 index 0000000000000..0b09920cf90a5 --- /dev/null +++ b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_5.json @@ -0,0 +1,1693 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "hive", + "env": "PROD", + "database": "hive" + }, + "name": "hive" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:hive" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Database" + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "hive", + "env": "PROD", + "database": "hive", + "schema": "db1" + }, + "name": "db1" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:hive" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.map_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "keyvalue", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "recordid", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.MapType": { + "keyType": "string", + "valueType": "string" + } + } + }, + "nativeDataType": "map", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"map\", \"key_type\": {\"type\": \"int\", \"native_data_type\": \"int\", \"_nullable\": true}, \"key_native_data_type\": \"int\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", + "transient_lastDdlTime": "1688395014", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test", + "create_date": "2023-07-03" + }, + "name": "map_test", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.union_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=struct].[type=union].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "union", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=int].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=double].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "double", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=array].[type=string].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "struct0", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=int].a", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=string].b", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", + "transient_lastDdlTime": "1688395014", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test", + "create_date": "2023-07-03" + }, + "name": "union_test", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.nested_struct_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "property_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "service", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct>\"}" + }, + { + "fieldPath": "service.type", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "service.provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct\"}" + }, + { + "fieldPath": "service.provider.name", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(50)", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"varchar(50)\", \"_nullable\": true}" + }, + { + "fieldPath": "service.provider.id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "tinyint", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"tinyint\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", + "transient_lastDdlTime": "1688395014", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test", + "create_date": "2023-07-03" + }, + "name": "nested_struct_test", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.array_struct_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "description": "id of property", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", + "nullable": true, + "description": "service types and providers", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "record" + ] + } + } + }, + "nativeDataType": "array>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array>>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "comment": "This table has array of structs", + "numFiles": "1", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "transient_lastDdlTime": "1688395011", + "rawDataSize": "32", + "numRows": "1", + "totalSize": "33", + "another.comment": "This table has no partitions", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test", + "create_date": "2023-07-03" + }, + "name": "array_struct_test", + "description": "This table has array of structs", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.struct_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "transient_lastDdlTime": "1688395008", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test", + "create_date": "2023-07-03" + }, + "name": "struct_test", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1._test_table_underscore", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "bar", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "numFiles": "0", + "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}", + "transient_lastDdlTime": "1688395008", + "rawDataSize": "0", + "numRows": "0", + "totalSize": "0", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore", + "create_date": "2023-07-03" + }, + "name": "_test_table_underscore", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.pokes", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "baz", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "bar", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "transient_lastDdlTime": "1688395005", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", + "create_date": "2023-07-03", + "partitioned_columns": "baz" + }, + "name": "pokes", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.array_struct_test_presto_view", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "dummy", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "is_view": "True" + }, + "name": "array_struct_test_presto_view", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "/* Presto View */", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "view" + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "/* Presto View */", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.array_struct_test_view", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "record" + ] + } + } + }, + "nativeDataType": "array>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array>>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "is_view": "True" + }, + "name": "array_struct_test_view", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "view" + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "urn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918" + }, + { + "id": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "urn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +} +] diff --git a/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py b/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py index 8828f0a92e40d..17e21f3790070 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py +++ b/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py @@ -53,12 +53,14 @@ def loaded_presto_on_hive(presto_on_hive_runner): @freeze_time(FROZEN_TIME) @pytest.mark.integration_batch_1 @pytest.mark.parametrize( - "mode,use_catalog_subtype,use_dataset_pascalcase_subtype,include_catalog_name_in_ids,test_suffix", + "mode,use_catalog_subtype,use_dataset_pascalcase_subtype,include_catalog_name_in_ids,simplify_nested_field_paths," + "test_suffix", [ - ("hive", False, False, False, "_1"), - ("presto-on-hive", True, True, False, "_2"), - ("hive", False, False, True, "_3"), - ("presto-on-hive", True, True, True, "_4"), + ("hive", False, False, False, False, "_1"), + ("presto-on-hive", True, True, False, False, "_2"), + ("hive", False, False, True, False, "_3"), + ("presto-on-hive", True, True, True, False, "_4"), + ("hive", False, False, False, True, "_5"), ], ) def test_presto_on_hive_ingest( @@ -71,6 +73,7 @@ def test_presto_on_hive_ingest( use_catalog_subtype, use_dataset_pascalcase_subtype, include_catalog_name_in_ids, + simplify_nested_field_paths, test_suffix, ): # Run the metadata ingestion pipeline. @@ -97,6 +100,7 @@ def test_presto_on_hive_ingest( "mode": mode, "use_catalog_subtype": use_catalog_subtype, "use_dataset_pascalcase_subtype": use_dataset_pascalcase_subtype, + "simplify_nested_field_paths": simplify_nested_field_paths, }, }, "sink": { From ace74fae050574db9bf337c562361f59a4544717 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Tue, 1 Aug 2023 17:38:48 -0400 Subject: [PATCH 220/222] fix(ui) Make multiple small updates to new search and browse (#8524) Co-authored-by: Aseem Bansal --- .../domain/DataProductsTab/DataProductsTab.tsx | 2 ++ .../src/app/preview/DefaultPreviewCard.tsx | 2 +- .../src/app/search/SearchFilterLabel.tsx | 4 +--- .../src/app/search/filters/FilterOption.tsx | 13 +++++++++++-- .../src/app/search/filters/OptionsDropdownMenu.tsx | 5 ++++- .../src/app/search/sidebar/EntityNode.tsx | 5 +++-- datahub-web-react/src/app/search/utils/constants.ts | 2 ++ 7 files changed, 24 insertions(+), 9 deletions(-) diff --git a/datahub-web-react/src/app/entity/domain/DataProductsTab/DataProductsTab.tsx b/datahub-web-react/src/app/entity/domain/DataProductsTab/DataProductsTab.tsx index 6f10c97423731..c834597e41a6d 100644 --- a/datahub-web-react/src/app/entity/domain/DataProductsTab/DataProductsTab.tsx +++ b/datahub-web-react/src/app/entity/domain/DataProductsTab/DataProductsTab.tsx @@ -57,6 +57,7 @@ export default function DataProductsTab() { const domainUrn = entityData?.urn || ''; const { data, loading } = useGetSearchResultsForMultipleQuery({ + skip: !domainUrn, variables: { input: { types: [EntityType.DataProduct], @@ -64,6 +65,7 @@ export default function DataProductsTab() { start, count: DEFAULT_PAGE_SIZE, orFilters: [{ and: [{ field: DOMAINS_FILTER_NAME, values: [domainUrn] }] }], + searchFlags: { skipCache: true }, }, }, }); diff --git a/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx b/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx index 46ee36809858f..0a7d16ade0ac0 100644 --- a/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx +++ b/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx @@ -294,7 +294,7 @@ export default function DefaultPreviewCard({ {deprecation?.deprecated && ( )} - {health && health.length && } + {health && health.length > 0 && } {externalUrl && ( { const entityRegistry = useEntityRegistry(); diff --git a/datahub-web-react/src/app/search/filters/FilterOption.tsx b/datahub-web-react/src/app/search/filters/FilterOption.tsx index c5ef6b45826e4..b112413efd845 100644 --- a/datahub-web-react/src/app/search/filters/FilterOption.tsx +++ b/datahub-web-react/src/app/search/filters/FilterOption.tsx @@ -7,11 +7,18 @@ import { EntityType, GlossaryNode, GlossaryTerm, Tag } from '../../../types.gene import { generateColor } from '../../entity/shared/components/styled/StyledTag'; import { ANTD_GRAY } from '../../entity/shared/constants'; import { useEntityRegistry } from '../../useEntityRegistry'; -import { PLATFORM_FILTER_NAME, TAGS_FILTER_NAME, TYPE_NAMES_FILTER_NAME } from '../utils/constants'; +import { + ENTITY_SUB_TYPE_FILTER_NAME, + MAX_COUNT_VAL, + PLATFORM_FILTER_NAME, + TAGS_FILTER_NAME, + TYPE_NAMES_FILTER_NAME, +} from '../utils/constants'; import { IconSpacer, Label } from './ActiveFilter'; import { isFilterOptionSelected, getFilterIconAndLabel, isAnyOptionSelected } from './utils'; import { capitalizeFirstLetterOnly } from '../../shared/textUtil'; import ParentNodes from './ParentNodes'; +import { formatNumber } from '../../shared/formatNumber'; const FilterOptionWrapper = styled.div<{ centerAlign?: boolean; addPadding?: boolean }>` display: flex; @@ -119,6 +126,8 @@ export default function FilterOption({ const isSubTypeFilter = field === TYPE_NAMES_FILTER_NAME; const isGlossaryTerm = entity?.type === EntityType.GlossaryTerm; const parentNodes: GlossaryNode[] = isGlossaryTerm ? (entity as GlossaryTerm).parentNodes?.nodes || [] : []; + // only entity type filters return 10,000 max aggs + const countText = count === MAX_COUNT_VAL && field === ENTITY_SUB_TYPE_FILTER_NAME ? '10k+' : formatNumber(count); function updateFilterValues() { if (isFilterOptionSelected(selectedFilterOptions, value)) { @@ -156,7 +165,7 @@ export default function FilterOption({ - {count} + {countText} {nestedOptions && nestedOptions.length > 0 && ( } diff --git a/datahub-web-react/src/app/search/filters/OptionsDropdownMenu.tsx b/datahub-web-react/src/app/search/filters/OptionsDropdownMenu.tsx index 321fedd57dfe9..9302b908d8a1b 100644 --- a/datahub-web-react/src/app/search/filters/OptionsDropdownMenu.tsx +++ b/datahub-web-react/src/app/search/filters/OptionsDropdownMenu.tsx @@ -4,6 +4,7 @@ import React from 'react'; import styled from 'styled-components/macro'; import { useEntityRegistry } from '../../useEntityRegistry'; import { SearchBar } from '../SearchBar'; +import { useEnterKeyListener } from '../../shared/useEnterKeyListener'; const StyledButton = styled(Button)` width: 100%; @@ -73,6 +74,8 @@ export default function OptionsDropdownMenu({ }: Props) { const entityRegistry = useEntityRegistry(); + useEnterKeyListener({ querySelectorToExecuteClick: '#updateFiltersButton' }); + return ( @@ -101,7 +104,7 @@ export default function OptionsDropdownMenu({ )} - + Update diff --git a/datahub-web-react/src/app/search/sidebar/EntityNode.tsx b/datahub-web-react/src/app/search/sidebar/EntityNode.tsx index 46310d4534f46..e04e4253dca13 100644 --- a/datahub-web-react/src/app/search/sidebar/EntityNode.tsx +++ b/datahub-web-react/src/app/search/sidebar/EntityNode.tsx @@ -8,7 +8,7 @@ import { formatNumber } from '../../shared/formatNumber'; import ExpandableNode from './ExpandableNode'; import EnvironmentNode from './EnvironmentNode'; import useAggregationsQuery from './useAggregationsQuery'; -import { ORIGIN_FILTER_NAME, PLATFORM_FILTER_NAME } from '../utils/constants'; +import { MAX_COUNT_VAL, ORIGIN_FILTER_NAME, PLATFORM_FILTER_NAME } from '../utils/constants'; import PlatformNode from './PlatformNode'; import SidebarLoadingError from './SidebarLoadingError'; import useToggle from '../../shared/useToggle'; @@ -28,6 +28,7 @@ const EntityNode = () => { const entityAggregation = useEntityAggregation(); const hasEnvironmentFilter = useHasFilterField(ORIGIN_FILTER_NAME); const { count } = entityAggregation; + const countText = count === MAX_COUNT_VAL ? '10k+' : formatNumber(count); const registry = useEntityRegistry(); const { trackToggleNodeEvent } = useSidebarAnalytics(); @@ -67,7 +68,7 @@ const EntityNode = () => { {registry.getCollectionName(entityType)} - {formatNumber(entityAggregation.count)} + {countText} diff --git a/datahub-web-react/src/app/search/utils/constants.ts b/datahub-web-react/src/app/search/utils/constants.ts index d65335f37109d..eecd18441e7a5 100644 --- a/datahub-web-react/src/app/search/utils/constants.ts +++ b/datahub-web-react/src/app/search/utils/constants.ts @@ -124,3 +124,5 @@ export const FilterModes = { } as const; export type FilterMode = typeof FilterModes[keyof typeof FilterModes]; + +export const MAX_COUNT_VAL = 10000; From e380f7d317bad20ccba7464fe9366d400f32e352 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Tue, 1 Aug 2023 15:31:24 -0700 Subject: [PATCH 221/222] feat(search): Allow aggregating on facets that are not explicitly part of default filter set (#8540) --- .../request/AggregationQueryBuilder.java | 38 +++++++++--- .../request/AggregationQueryBuilderTest.java | 61 ++++++++++++++++++- 2 files changed, 88 insertions(+), 11 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java index 2e080747a899d..d02aa960b4868 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java @@ -21,13 +21,15 @@ public class AggregationQueryBuilder { private final SearchConfiguration _configs; - private final Set _facetFields; + private final Set _defaultFacetFields; + private final Set _allFacetFields; public AggregationQueryBuilder( @Nonnull final SearchConfiguration configs, @Nonnull final List annotations) { this._configs = Objects.requireNonNull(configs, "configs must not be null"); - this._facetFields = getFacetFields(annotations); + this._defaultFacetFields = getDefaultFacetFields(annotations); + this._allFacetFields = getAllFacetFields(annotations); } /** @@ -44,19 +46,28 @@ public List getAggregations(@Nullable List facets) { final Set facetsToAggregate; if (facets != null) { facets.stream().filter(f -> !isValidAggregate(f)).forEach(facet -> { - log.warn(String.format("Provided facet for search filter aggregations that doesn't exist. Provided: %s; Available: %s", facet, _facetFields)); + log.warn(String.format("Requested facet for search filter aggregations that isn't part of the default filters. Provided: %s; Available: %s", facet, + _defaultFacetFields)); }); facetsToAggregate = facets.stream().filter(this::isValidAggregate).collect(Collectors.toSet()); } else { - facetsToAggregate = _facetFields; + facetsToAggregate = _defaultFacetFields; } return facetsToAggregate.stream().map(this::facetToAggregationBuilder).collect(Collectors.toList()); } - private Set getFacetFields(final List annotations) { + private Set getDefaultFacetFields(final List annotations) { Set facets = annotations.stream() - .flatMap(annotation -> getFacetFieldsFromAnnotation(annotation).stream()) + .flatMap(annotation -> getDefaultFacetFieldsFromAnnotation(annotation).stream()) + .collect(Collectors.toSet()); + facets.add(INDEX_VIRTUAL_FIELD); + return facets; + } + + private Set getAllFacetFields(final List annotations) { + Set facets = annotations.stream() + .flatMap(annotation -> getAllFacetFieldsFromAnnotation(annotation).stream()) .collect(Collectors.toSet()); facets.add(INDEX_VIRTUAL_FIELD); return facets; @@ -64,7 +75,7 @@ private Set getFacetFields(final List annotations) private boolean isValidAggregate(final String inputFacet) { Set facets = Set.of(inputFacet.split(AGGREGATION_SEPARATOR_CHAR)); - return facets.size() > 0 && _facetFields.containsAll(facets); + return facets.size() > 0 && _allFacetFields.containsAll(facets); } private AggregationBuilder facetToAggregationBuilder(final String inputFacet) { @@ -97,7 +108,7 @@ private String getAggregationField(final String facet) { return ESUtils.toKeywordField(facet, false); } - List getFacetFieldsFromAnnotation(final SearchableAnnotation annotation) { + List getDefaultFacetFieldsFromAnnotation(final SearchableAnnotation annotation) { final List facetsFromAnnotation = new ArrayList<>(); if (annotation.isAddToFilters()) { facetsFromAnnotation.add(annotation.getFieldName()); @@ -107,4 +118,13 @@ List getFacetFieldsFromAnnotation(final SearchableAnnotation annotation) } return facetsFromAnnotation; } -} + + List getAllFacetFieldsFromAnnotation(final SearchableAnnotation annotation) { + final List facetsFromAnnotation = new ArrayList<>(); + facetsFromAnnotation.add(annotation.getFieldName()); + if (annotation.getHasValuesFieldName().isPresent()) { + facetsFromAnnotation.add(annotation.getHasValuesFieldName().get()); + } + return facetsFromAnnotation; + } +} \ No newline at end of file diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilderTest.java index 76160eb29af4e..10b4ee42b1a71 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilderTest.java @@ -1,11 +1,14 @@ package com.linkedin.metadata.search.elasticsearch.query.request; +import com.google.common.collect.ImmutableSet; import com.linkedin.metadata.config.search.SearchConfiguration; import com.google.common.collect.ImmutableList; import com.linkedin.metadata.models.annotation.SearchableAnnotation; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.testng.Assert; import org.testng.annotations.Test; @@ -14,7 +17,7 @@ public class AggregationQueryBuilderTest { @Test - public void testGetAggregationsHasFields() { + public void testGetDefaultAggregationsHasFields() { SearchableAnnotation annotation = new SearchableAnnotation( "test", @@ -43,7 +46,7 @@ public void testGetAggregationsHasFields() { } @Test - public void testGetAggregationsFields() { + public void testGetDefaultAggregationsFields() { SearchableAnnotation annotation = new SearchableAnnotation( "test", @@ -70,4 +73,58 @@ public void testGetAggregationsFields() { Assert.assertTrue(aggs.stream().anyMatch(agg -> agg.getName().equals("test"))); } + + @Test + public void testGetSpecificAggregationsHasFields() { + + SearchableAnnotation annotation1 = new SearchableAnnotation( + "test1", + SearchableAnnotation.FieldType.KEYWORD, + true, + true, + false, + false, + Optional.empty(), + Optional.of("Has Test"), + 1.0, + Optional.of("hasTest1"), + Optional.empty(), + Collections.emptyMap() + ); + + SearchableAnnotation annotation2 = new SearchableAnnotation( + "test2", + SearchableAnnotation.FieldType.KEYWORD, + true, + true, + false, + false, + Optional.of("Test Filter"), + Optional.empty(), + 1.0, + Optional.empty(), + Optional.empty(), + Collections.emptyMap() + ); + + SearchConfiguration config = new SearchConfiguration(); + config.setMaxTermBucketSize(25); + + AggregationQueryBuilder builder = new AggregationQueryBuilder( + config, ImmutableList.of(annotation1, annotation2)); + + // Case 1: Ask for fields that should exist. + List aggs = builder.getAggregations( + ImmutableList.of("test1", "test2", "hasTest1") + ); + Assert.assertEquals(aggs.size(), 3); + Set facets = aggs.stream().map(AggregationBuilder::getName).collect(Collectors.toSet()); + Assert.assertEquals(ImmutableSet.of("test1", "test2", "hasTest1"), facets); + + // Case 2: Ask for fields that should NOT exist. + aggs = builder.getAggregations( + ImmutableList.of("hasTest2") + ); + Assert.assertEquals(aggs.size(), 0); + } } From 4f9fc671dcea03cbb22b7c0e02e29bdf88ba955f Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 1 Aug 2023 19:59:30 -0500 Subject: [PATCH 222/222] fix(test): increase siblings.js test stability (#8542) --- datahub-web-react/src/app/lineage/LineageEntityNode.tsx | 9 ++++++--- .../tests/cypress/cypress/e2e/siblings/siblings.js | 8 +++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/datahub-web-react/src/app/lineage/LineageEntityNode.tsx b/datahub-web-react/src/app/lineage/LineageEntityNode.tsx index 5b39d6aaf19c3..4526e3a225ce2 100644 --- a/datahub-web-react/src/app/lineage/LineageEntityNode.tsx +++ b/datahub-web-react/src/app/lineage/LineageEntityNode.tsx @@ -132,6 +132,10 @@ export default function LineageEntityNode({ areColumnsCollapsed, ); + const entityName = + capitalizeFirstLetterOnly(node.data.subtype) || + (node.data.type && entityRegistry.getEntityName(node.data.type)); + return ( {unexploredHiddenChildren && (isHovered || isSelected) ? ( @@ -335,9 +339,8 @@ export default function LineageEntityNode({ {' '} |{' '} - - {capitalizeFirstLetterOnly(node.data.subtype) || - (node.data.type && entityRegistry.getEntityName(node.data.type))} + + {entityName} {expandTitles ? ( diff --git a/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js b/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js index b7d48703992e2..00de08e77a185 100644 --- a/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js +++ b/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js @@ -113,9 +113,11 @@ describe('siblings', () => { cy.clickOptionWithTestId('compress-lineage-toggle'); // check the subtypes - cy.get('text:contains(View)').should('have.length', 2); - cy.get('text:contains(Table)').should('have.length', 0); - cy.get('text:contains(Seed)').should('have.length', 1); + cy.get('[data-testid="Seed"]').should('have.length', 1); + // center counts twice since we secretely render two center nodes, plus the downstream bigquery + cy.get('[data-testid="View"]').should('have.length', 3); + cy.get('[data-testid="Table"]').should('have.length', 0); + // check the names cy.get('text:contains(raw_orders)').should('have.length', 1);